diff --git a/MODULE.bazel b/MODULE.bazel index 8bf7bb2c1c1a230126f6261b7f68ab18e5ff5101..4a87144511fc31ecd0867a76ab3d62748053af04 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,15 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.18") bazel_dep(name = "ape", version = "1.0.0-beta.17") bazel_dep(name = "rules_go", version = "0.48.1") +bazel_dep(name = "gazelle", version = "0.31.0") + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//curl/upload:go.mod") +use_repo( + go_deps, "com_github_google_shlex") + +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.23.4") export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export") use_repo(export, "ape-curl") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a7d227e5280f159cdb033d6a5f8283c35708a2b8..585372142942af9e0f8c519931801c1a3762505e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -28,6 +28,9 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.31.0/MODULE.bazel": "0319690246f72d0b5d596724a0ea0da2fd823905643a042c95bc2c420438ddae", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", @@ -52,6 +55,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", diff --git a/README.md b/README.md index d88cecd0df9f9ad082345bc1ca57a06a373839f4..e7c9f5bf5a188b12badc1a43f0e55e1b755df6c8 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,6 @@ Add the following to `MODULE.bazel`: bazel_dep(module_name = "rules_curl", version = "0.0.0") ``` -## Providers - -### ManifestInfo - -Encapsulates the file to upload and its URL template (see [ManifestInfo][manifest_info]) - -Members: - -- **file**: File -- **url**: string - -### ManifestsInfo - -The dependency set of `ManifestInfo`s - -Members: - -- **manifests**: depset of `ManifestInfo`s - ## Usage ### curl_upload_file @@ -44,22 +25,3 @@ curl_upload_file( ) ``` - -### curl_upload_manifests - -```py -load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") - -curl_upload_manifests( - name = "upload", - srcs = [ - ":fixture_1", - ":fixture_2", - ], - url = "https://test.case", -) -``` - -> Note: `fixture_1` and `fixture_2` have to provide either `ManifestInfo` or `ManifestsInfo` - -[manifest_info]: curl/upload/ManifestInfo.bzl diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel deleted file mode 100644 index d22934d953836277f571b50569055c85f8bb8883..0000000000000000000000000000000000000000 --- a/curl/template/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary") - -go_binary( - name = "template", - srcs = ["template.go"], - pure = "on", - visibility = ["//curl/upload:__subpackages__"], -) diff --git a/curl/template/template.go b/curl/template/template.go deleted file mode 100644 index 1a0d740bc7d8de6272ea472d16561047f3de1bc4..0000000000000000000000000000000000000000 --- a/curl/template/template.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - "bytes" - "flag" - "fmt" - "html/template" - "log" - "net/url" - "path/filepath" - "strconv" - "strings" -) - -var manifest ManifestInfo = ManifestInfo{ - URL: URLInfo{}, - File: FileInfo{}, -} - -type StringInfo string - -func (s StringInfo) String() string { - return string(s) -} - -func (s *StringInfo) Set(value string) error { - buf := new(bytes.Buffer) - - tmpl, err := template.New("templ").Parse(value) - if err != nil { - log.Fatal(err) - } - - err = tmpl.Execute(buf, manifest) - if err != nil { - log.Fatal(err) - } - - *s = StringInfo(buf.String()) - return nil -} - -type FileInfo struct { - Dirname StringInfo - Stem StringInfo - Extension StringInfo -} - -func (f FileInfo) Basename() string { - extension := f.Extension.String() - stem := f.Stem.String() - if extension == "" { - return stem - } - return stem + "." + extension -} - -func (f FileInfo) Path() string { - return f.Dirname.String() + "/" + f.Basename() -} - -func (f FileInfo) String() string { - return f.Path() -} - -func (f *FileInfo) Set(value string) error { - clean := filepath.Clean(value) - dirname := filepath.Dir(clean) - f.Dirname = StringInfo(dirname) - - extension := filepath.Ext(value) - nodot := strings.TrimPrefix(extension, ".") - f.Extension = StringInfo(nodot) - - base := filepath.Base(value) - stem := strings.TrimSuffix(base, extension) - f.Stem = StringInfo(stem) - return nil -} - -type URLInfo struct { - Username *StringInfo - Password *StringInfo - Host StringInfo - Pathname StringInfo - Protocol StringInfo -} - -func (u URLInfo) split() (string, *int) { - host := u.Host.String() - index := strings.LastIndexByte(host, ':') - - if index == -1 { - return host, nil - } - - hostname := host[:index] - port, err := strconv.Atoi(host[index+1:]) - if err != nil { - log.Fatal(err) - } - return hostname, &port -} - -func (u URLInfo) Hostname() string { - hostname, _ := u.split() - return hostname -} - -func (u URLInfo) Port() *int { - _, port := u.split() - return port -} - -func (u URLInfo) Origin() string { - return u.Protocol.String() + "//" + u.Host.String() -} - -func (u URLInfo) Auth() (auth string) { - if u.Username != nil { - auth += string(*u.Username) - } - if u.Password != nil { - auth += ":" + string(*u.Password) - } - return -} - -func (u URLInfo) Href() (href string) { - href += u.Protocol.String() + "//" - if auth := u.Auth(); auth != "" { - href += auth + "@" - } - href += u.Host.String() - href += u.Pathname.String() - return -} - -func (u URLInfo) String() string { - return u.Href() -} - -func (u *URLInfo) Set(value string) error { - parsed, err := url.Parse(value) - if err != nil { - return err - } - if username := parsed.User.Username(); username != "" { - info := StringInfo(username) - u.Username = &info - } - if password, set := parsed.User.Password(); set { - info := StringInfo(password) - u.Password = &info - } - u.Host = StringInfo(parsed.Host) - u.Pathname = StringInfo(parsed.Path) - u.Protocol = StringInfo(parsed.Scheme + ":") - return err -} - -type ManifestInfo struct { - URL URLInfo - File FileInfo -} - -func main() { - flag.Var(&manifest.File, "file", "The file path to use for templating") - flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file") - flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file") - flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file") - flag.Var(&manifest.URL, "url", "URL to use for templating") - flag.Var(&manifest.URL.Host, "host", "The domain name of the URL") - flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL") - flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error { - return manifest.URL.Pathname.Set(s) - }) - flag.Func("origin", "The origin of the represented URL.", func(s string) error { - index := strings.Index(s, "//") - - if index == -1 { - manifest.URL.Protocol = StringInfo(s) - return nil - } - - manifest.URL.Host = StringInfo(s[index+2:]) - manifest.URL.Protocol = StringInfo(s[:index]) - - return nil - }) - t := flag.String("template", "{{.URL.Href}}/{{.File.Path}}", "The Go template to render") - - flag.Parse() - - tmpl, err := template.New("url").Parse(*t) - if err != nil { - log.Fatal(err) - } - - buf := new(bytes.Buffer) - - err = tmpl.Execute(buf, manifest) - if err != nil { - log.Fatal(err) - } - - _, err = url.Parse(buf.String()) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("%s", buf.String()) -} diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 04169ed89494ac539e6d5f36a04c011d1a9b2080..ed6df268aad270706e2158944d2ea551ccd23cab 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -1,22 +1,17 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) +load("@rules_go//go:def.bzl", "go_binary") -alias( - name = "script", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_manifests`", - ), - visibility = ["//curl/upload:__subpackages__"], -) -sh_binary( - name = "csv", - srcs = ["csv.sh"], +go_binary( + name = "upload", + srcs = ["upload.go"], visibility = ["//curl/upload:__subpackages__"], + deps = [ + "@rules_go//go/runfiles:go_default_library", + "@com_github_google_shlex//:shlex" + ], + pure = "on" ) + +exports_files([ + "go.mod", +]) diff --git a/curl/upload/ManifestInfo.bzl b/curl/upload/ManifestInfo.bzl deleted file mode 100644 index 2694446a2c11ffe591b22c58724d2eb1d95d6c25..0000000000000000000000000000000000000000 --- a/curl/upload/ManifestInfo.bzl +++ /dev/null @@ -1,58 +0,0 @@ -load("@bazel_skylib//lib:types.bzl", "types") - -visibility("//...") - -def init(file, url): - """ - Initializes a `CurlUploadManifestInfo` provider. - - To be used with `curl_upload_manifests` - - Args: - file: The file to upload. - url: A Starlark string template for the URL, which can include: - - {{.URL.Href}}: the provided URL - - {{.URL.Auth}}: the user:password auth - - {{.URL.Origin}}: the origin of the represented URL - - {{.URL.Port}}: the port number of the URL - - {{.URL.Host}}: the hostname followed by the port if not empty - - {{.URL.Hostname}}: the domain name of the URL - - {{.URL.Protocol}}: the protocol scheme of the URL - - {{.URL.Pathname}}: a location in a hierarchical structure - - {{.File.Path}}: the full path of the file - - {{.File.Basename}}: the basename of the file - - {{.File.Dirname}}: the directory of the file - - {{.File.Stem}}: the file name without extension - - {{.File.Extension}}: the extension of the file - - Returns: - A mapping of keywords for the `curl_upload_manifest_info` raw constructor. - """ - if type(file) != "File": - fail("`CurlUploadManifestInfo.file` must be a `file`: {}".format(file)) - - if file.is_directory: - fail("`CurlUploadManifestInfo.file` must not be a directory: {}".format(file)) - - if not types.is_string(url): - fail("`CurlUploadManifestInfo.url` must be a `str`: {}".format(url)) - - if url.find(",") != -1: - fail("`CurlUploadManifestInfo.url` must not have comma `,` sign: {}".format(url)) - - return { - "file": file, - "url": url, - } - -CurlUploadManifestInfo, curl_upload_manifest_info = provider( - "A file to upload with cURL.", - fields = ["file", "url"], - init = init, -) - -# Provide some convenience imports -ManifestInfo = CurlUploadManifestInfo -manifest_info = curl_upload_manifest_info -Info = CurlUploadManifestInfo -info = curl_upload_manifest_info diff --git a/curl/upload/ManifestsInfo.bzl b/curl/upload/ManifestsInfo.bzl deleted file mode 100644 index 800e6d3eb2cde58822ea9a6de87cc43dc4225b26..0000000000000000000000000000000000000000 --- a/curl/upload/ManifestsInfo.bzl +++ /dev/null @@ -1,34 +0,0 @@ -load("@bazel_skylib//lib:types.bzl", "types") - -visibility("//...") - -def init(manifests): - """ - Initializes a `CurlUploadManifestsInfo` provider. - - To be used with `curl_upload_manifests` - - Args: - manifests: The dependency set of `CurlUploadManifestInfo`s - - Returns: - A mapping of keywords for the `curl_upload_manifests_info` raw constructor. - """ - if not types.is_depset(manifests): - fail("`CurlUploadManifestsInfo.url` must be a `depset`: {}".format(manifests)) - - return { - "manifests": manifests, - } - -CurlUploadManifestsInfo, curl_upload_manifests_info = provider( - "Files to upload with cURL.", - fields = ["manifests"], - init = init, -) - -# Provide some convenience imports -ManifestsInfo = CurlUploadManifestsInfo -manifests_info = curl_upload_manifests_info -Info = CurlUploadManifestsInfo -info = curl_upload_manifests_info diff --git a/curl/upload/csv.sh b/curl/upload/csv.sh deleted file mode 100755 index 39301d041a6c65fcd5deb252a8fa7078d75419a3..0000000000000000000000000000000000000000 --- a/curl/upload/csv.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env sh - -set -o errexit -o nounset - -OUT="${1}" -shift -readonly OUT - -for ARG in "${@}"; do - if test -n "${ARG}"; then - printf >>"${OUT}" '%s\n' "${ARG}" - fi -done diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index ddad0e119ba646ea185c54f979bc89fd0ca7da75..e36cd4caf3026d4af5139f1aef6c92842a5a6559 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -1,5 +1,3 @@ -load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info") - visibility("//curl/...") DOC = """Upload a file to a URL endpoint with cURL. @@ -8,7 +6,6 @@ DOC = """Upload a file to a URL endpoint with cURL. file( name = "upload_file", src = ":data", - dst = "data" url = "https://host.name.to.upload", ) ``` @@ -20,12 +17,8 @@ ATTRS = { mandatory = True, allow_single_file = True, ), - "dst": attr.string( - doc = "The filename to upload as.", - mandatory = True, - ), "url": attr.string( - doc = "URL endpoint for file upload.", + doc = "URL endpoint for file upload. Subject to 'Make variable' expansion.", mandatory = True, ), "retry": attr.int( @@ -36,66 +29,52 @@ ATTRS = { doc = "The seconds to wait before attempting a upload retry.", default = 1, ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", + "_upload": attr.label( + default = "//curl/upload", allow_single_file = True, executable = True, - ), - "_csv": attr.label( - doc = "CSV tool", - default = "//curl/upload:csv", cfg = "exec", - executable = True, - ), + ), } +def _runfile(label, file): + path = file.short_path + if path.startswith("../"): + return path.removeprefix("../") + return "{}/{}".format(label.workspace_name or "_main", path) + def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] - csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) - href = ctx.attr.url.rstrip("/").replace(",", "%2C") - - args = ctx.actions.args() - # SRC, DST, TEMPLATE, URL - args.add("{},{},{},{}".format(ctx.file.src.short_path, ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) - - ctx.actions.run( - outputs = [csv], - inputs = [ctx.file.src], - arguments = [csv.path, args], - executable = ctx.executable._csv, - mnemonic = "PrepareUploadCSV", - ) + if ctx.file.src.is_directory: + fail("'src' must be a file not a directory.") - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) + # Do the 'Make variable' expansion on the URL. + url = ctx.attr.url + for k, v in ctx.var.items(): + url = url.replace("$({})".format(k), v) - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - substitutions.add("{{directory}}", str(ctx.file.src.short_path)) + arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) + args = ctx.actions.args() + args.add("src", _runfile(ctx.file.src.owner, ctx.file.src)) + args.add("url", url) + args.add("curl", _runfile(curl.executable.owner, curl.executable)) + args.add("retry", str(ctx.attr.retry)) + args.add("retry-delay", str(ctx.attr.retry_delay)) + ctx.actions.write(output = arguments, content = args) - ctx.actions.expand_template( - template = ctx.file._script, + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( output = executable, - computed_substitutions = substitutions, + target_file = ctx.executable._upload, is_executable = True, ) - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, ctx.file.src, csv]) - runfiles = runfiles.merge(ctx.attr.src.default_runfiles) + files = depset([executable, arguments]) + root_symlinks = {"upload.args": arguments} + + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, ctx.file.src], root_symlinks = root_symlinks) runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) - runfiles = runfiles.merge(ctx.attr._csv.default_runfiles) return DefaultInfo( executable = executable, diff --git a/curl/upload/go.mod b/curl/upload/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..8d08116554bd7a45bb21cc97eddeb5de0d110d2f --- /dev/null +++ b/curl/upload/go.mod @@ -0,0 +1,8 @@ +module curl_runner + +go 1.19 + + +require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +) diff --git a/curl/upload/go.sum b/curl/upload/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..a14fb8616dccc15ae11fe1f634c47cad3b35f8bc --- /dev/null +++ b/curl/upload/go.sum @@ -0,0 +1,2 @@ +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= diff --git a/curl/upload/manifest/BUILD.bazel b/curl/upload/manifest/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/curl/upload/manifest/defs.bzl b/curl/upload/manifest/defs.bzl deleted file mode 100644 index ce7ef89794a233248c70386e6cf35ccc3bcf225a..0000000000000000000000000000000000000000 --- a/curl/upload/manifest/defs.bzl +++ /dev/null @@ -1,5 +0,0 @@ -load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info") - -visibility("public") - -CurlUploadManifestInfo = _Manifest diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/curl/upload/manifests/defs.bzl b/curl/upload/manifests/defs.bzl deleted file mode 100644 index 0da0eba9e30c6eabdb15119927a3d343282a140a..0000000000000000000000000000000000000000 --- a/curl/upload/manifests/defs.bzl +++ /dev/null @@ -1,9 +0,0 @@ -load(":rule.bzl", _manifests = "manifests") -load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info") -load("//curl/upload:ManifestsInfo.bzl", _Manifests = "Info") - -visibility("public") - -curl_upload_manifests = _manifests -CurlUploadManifestInfo = _Manifest -CurlUploadManifestsInfo = _Manifests diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl deleted file mode 100644 index 713ebd9cb1cd3b09b393f663cfc5eb946186cdd5..0000000000000000000000000000000000000000 --- a/curl/upload/manifests/rule.bzl +++ /dev/null @@ -1,122 +0,0 @@ -load("//curl/upload:ManifestInfo.bzl", "ManifestInfo") -load("//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") - -visibility("//curl/...") - -DOC = """Upload bunch of files to a URL endpoint with cURL. - -The `srcs` must provide `ManifestInfo` or `ManifestsInfo`. - -```py -file( - name = "upload_files", - srcs = [ - ":data", - ], - url = "https://host.name.to.upload", -) -``` -""" - -ATTRS = { - "srcs": attr.label_list( - doc = "Files to be uploaded.", - mandatory = True, - providers = [ - [ManifestInfo], - [ManifestsInfo], - ], - allow_files = False, - ), - "url": attr.string( - doc = "URL endpoint for files to upload.", - mandatory = True, - ), - "retry": attr.int( - doc = "The number of retry attempts for every file.", - default = 3, - ), - "retry_delay": attr.int( - doc = "The seconds to wait before attempting an upload retry.", - default = 1, - ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", - allow_single_file = True, - executable = True, - ), - "_csv": attr.label( - doc = "CSV tool", - default = "//curl/upload:csv", - cfg = "exec", - executable = True, - ), -} - -def implementation(ctx): - curl = ctx.toolchains["//curl/toolchain/curl:type"] - - csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) - manifests = depset( - direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], - transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], - ) - - href = ctx.attr.url.rstrip("/").replace(",", "%2C") - - def _to_string(m): - # SRC, DST, TEMPLATE, URL - return "{},{},{},{}".format(m.file.short_path, m.file.short_path, m.url, href) - - args = ctx.actions.args() - args.add_all(manifests, map_each = _to_string, allow_closure = True) - - ctx.actions.run( - outputs = [csv], - arguments = [csv.path, args], - executable = ctx.executable._csv, - mnemonic = "PrepareUploadCSV", - ) - - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - - ctx.actions.expand_template( - template = ctx.file._script, - output = executable, - computed_substitutions = substitutions, - is_executable = True, - ) - - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, csv, ctx.executable._template] + [m.file for m in manifests.to_list()]) - runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) - - return DefaultInfo( - executable = executable, - files = files, - runfiles = runfiles, - ) - -curl_upload_manifests = rule( - doc = DOC, - attrs = ATTRS, - implementation = implementation, - toolchains = ["//curl/toolchain/curl:type"], - executable = True, -) - -manifests = curl_upload_manifests diff --git a/curl/upload/nt.tmpl.bat b/curl/upload/nt.tmpl.bat deleted file mode 100644 index d522214374991056aada3f19d730b97c81ba019a..0000000000000000000000000000000000000000 --- a/curl/upload/nt.tmpl.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -# TODO: implement Windows Batch for `curl_upload` -exit /b 121 diff --git a/curl/upload/posix.tmpl.sh b/curl/upload/posix.tmpl.sh deleted file mode 100644 index 05687f3e7f2eaf57cb53c9e44dfe50e1fac7f66d..0000000000000000000000000000000000000000 --- a/curl/upload/posix.tmpl.sh +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env sh - -# Strict shell -set -o errexit -o nounset - -# Runfiles location -rlocation() ( - readonly FILEPATH="${1}" - - if test -e "${FILEPATH}"; then - printf '%s' "${FILEPATH}" - return - fi - - readonly RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" - - if test -e "${RUNFILES_DIR}/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/${FILEPATH#../}" - return - fi - - if test -e "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/"*"/${FILEPATH#../}" - return - fi - - printf >&2 "No runfile found: %s\n" "${FILEPATH}" - exit 1 -) - -# Bazel substitutions -CURL="$(rlocation "{{curl}}")" -CSV="$(rlocation "{{csv}}")" -TEMPLATE="$(rlocation "{{template}}")" -RETRY="{{retry}}" -RETRY_DELAY="{{retry_delay}}" -readonly CURL CSV TEMPLATE RETRY RETRY_DELAY - -# Uploads a file -upload() { - printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" - - # Do the upload - "${CURL}" \ - --netrc \ - --location \ - --progress-bar \ - --retry "${RETRY}" \ - --retry-delay "${RETRY_DELAY}" \ - --upload-file "$(rlocation "${1}")" \ - "${2}" -} - -while IFS=, read -r SRC DST TMPL URL; do - FULL_DST=$( - "${TEMPLATE}" \ - --url "${URL}" \ - --template "${TMPL}" \ - --file "${DST}" \ - ${@} - ) - upload "${SRC}" "${FULL_DST}" -done <"${CSV}" diff --git a/curl/upload/upload.go b/curl/upload/upload.go new file mode 100644 index 0000000000000000000000000000000000000000..bf3dcce3fcb51b847d6f97a643f185d0c9b69ed8 --- /dev/null +++ b/curl/upload/upload.go @@ -0,0 +1,107 @@ +package main + +import ( + "flag" + "log" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/google/shlex" +) + +type RunfileVar string + +func (r RunfileVar) String() string { + return string(r) +} + +func (r *RunfileVar) Set(value string) error { + runfile, err := runfiles.Rlocation(value) + if err != nil { + log.Fatal(err) + } + + *r = RunfileVar(runfile) + return nil +} + +func upload(curl string, src string, dest string, retry uint64, retry_delay uint64) error { + args := []string{ + "--netrc", + "--location", + "--progress-bar", + "--retry", strconv.FormatUint(retry, 10), + "--retry-delay", strconv.FormatUint(retry_delay, 10), + "--upload-file", src, + "--url", dest, + } + cmd := exec.Command(curl, args...) + + cmd.Stdout = os.Stdout + + err := cmd.Run() + if err != nil { + return err + } + + return nil +} + +func setArgs(args_file string) error { + args_byt, err := os.ReadFile(args_file) + if err != nil { + return err + } + + args := strings.Split(string(args_byt), "\n") + // Remove ending newline + args = args[:len(args)-1] + + for i, _ := range args { + if i%2 != 0 { + unquoted, err := shlex.Split(args[i]) + err = flag.Set(args[i-1], unquoted[0]) + if err != nil { + return err + } + } + } + + return nil +} + +func main() { + var curl RunfileVar + var retry uint64 + var retry_delay uint64 + var src RunfileVar + var url string + + flag.Var(&curl, "curl", "The path to the curl bin") + flag.Uint64Var(&retry, "retry", 0, "The number of times to retry the request") + flag.Uint64Var(&retry_delay, "retry-delay", 1, "The number of seconds to wait before retrying the request") + + flag.Var(&src, "src", "Path to the source file") + flag.StringVar(&url, "url", "", "The URL to upload to") + + args_file, err := runfiles.Rlocation("upload.args") + if err != nil { + log.Fatal(err) + } + + err = setArgs(args_file) + if err != nil { + log.Fatal(err) + } + + flag.Parse() + + err = upload(curl.String(), src.String(), url, retry, retry_delay) + if err != nil { + log.Fatal(err) + } + +} diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index 5c6a6de5f9eb78f495606447e9a62fc7e1d1e719..cd70dcf724795f11977960f79ca8da6bf79e9e72 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -29,6 +29,9 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.31.0/MODULE.bazel": "0319690246f72d0b5d596724a0ea0da2fd823905643a042c95bc2c420438ddae", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", @@ -58,6 +61,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.6/MODULE.bazel": "5af9b4750785ee7603e4616a7463e87cb3ef016ec86da8cb6d592a33c735b6c0", "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.6/source.json": "58c2d1082dce218af90bcb70eddcb3b6b011dbe9168134614022f825f722a8db", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", diff --git a/e2e/mock/manifest/BUILD.bazel b/e2e/mock/manifest/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl deleted file mode 100644 index 8dcd73885bf99b2aa46cf7dad44b530130c4fe4c..0000000000000000000000000000000000000000 --- a/e2e/mock/manifest/rule.bzl +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_curl//curl/upload/manifest:defs.bzl", ManifestInfo = "CurlUploadManifestInfo") - -DOC = """Mock rule with output `ManifestInfo` provider. - -```py -mock_manifest( - name = "mock_file", -) -``` -""" - -def _impl(ctx): - out = ctx.actions.declare_file("{}.out".format(ctx.attr.name)) - - ctx.actions.write(out, "hello test", is_executable = False) - - return [ - DefaultInfo( - files = depset([out]), - ), - ManifestInfo( - file = out, - url = "{{.URL.Href}}/{{.File.Path}}", - ), - ] - -mock_manifest = rule( - doc = DOC, - implementation = _impl, - provides = [ - ManifestInfo, - ], -) diff --git a/e2e/mock/manifests/BUILD.bazel b/e2e/mock/manifests/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl deleted file mode 100644 index d786e4e220f14bd760f63ddcfb5787b9b821f8c0..0000000000000000000000000000000000000000 --- a/e2e/mock/manifests/rule.bzl +++ /dev/null @@ -1,56 +0,0 @@ -load("@rules_curl//curl/upload/manifests:defs.bzl", ManifestsInfo = "CurlUploadManifestsInfo", ManifestInfo = "CurlUploadManifestInfo") - -DOC = """Mock rule with output `ManifestsInfo` provider. - -```py -mock_manifests( - name = "mock_files", - outs = [ - "file_1.out", - "file_2.out", - ], - deps = [ - "my_dep", - ], -) -``` -""" - -ATTRS = { - "outs": attr.string_list( - default = [], - mandatory = True, - ), - "deps": attr.label_list(allow_files = True), -} - -def _impl(ctx): - outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] - - for f in outs: - ctx.actions.write(f, "hello test", is_executable = False) - - files = depset(outs) - - manifests = [ManifestInfo(file = o, url = "{{.URL.Href}}/{{.File.Path}}") for o in outs] - manifests.extend([dep[ManifestInfo] for dep in ctx.attr.deps if ManifestInfo in dep]) - - transitive = [dep[ManifestsInfo].manifests for dep in ctx.attr.deps if ManifestsInfo in dep] - - return [ - DefaultInfo( - files = files, - ), - ManifestsInfo( - manifests = depset(direct = manifests, transitive = transitive), - ), - ] - -mock_manifests = rule( - doc = DOC, - attrs = ATTRS, - implementation = _impl, - provides = [ - ManifestsInfo, - ], -) diff --git a/e2e/upload/file/BUILD.bazel b/e2e/upload/file/BUILD.bazel index 0cdb0437a9cedcb31eb5e4771498bb6352845b60..d3e917f724142107663eee6ae7e0b43fd6476622 100644 --- a/e2e/upload/file/BUILD.bazel +++ b/e2e/upload/file/BUILD.bazel @@ -1,53 +1,78 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -curl_upload_file( - name = "upload", - testonly = True, - src = ":fixture.txt", - dst = "directory/fixture.txt", - url = "https://test.case", +genrule( + name = "file_to_upload", + outs = [ + "input.txt", + ], + cmd = "echo TEST > $@", ) genrule( - name = "execute", - testonly = True, + name = "file_to_upload_no_extension", outs = [ - "upload.out", + "input", ], - cmd = "./$(location :upload) > $@", - tools = [":upload"], + cmd = "echo TEST > $@", ) -diff_file_test( - name = "test", - size = "small", - a = ":fixture.txt", - b = ":execute", +curl_upload_file( + name = "upload_basic", + testonly = True, + src = ":file_to_upload", + url = "https://test.case/directory/fixture.txt", ) - curl_upload_file( name = "upload_no_extension", testonly = True, - src = ":fixture.txt", - dst = "directory/fixture", - url = "https://test.case", + src = ":file_to_upload_no_extension", + url = "https://test.case/directory/fixture", ) -genrule( - name = "execute_no_extension", +string_flag( + name = "injection", + build_setting_default = "injection/", + make_variable = "INJECT", + visibility = ["//visibility:public"], +) + +curl_upload_file( + name = "upload_injection", testonly = True, - outs = [ - "upload_no_extension.out", - ], - cmd = "./$(location :upload_no_extension) > $@", - tools = [":upload_no_extension"], + src = ":file_to_upload", + toolchains = [":injection"], + url = "https://test.case/directory/$(INJECT)fixture.txt", ) -diff_file_test( - name = "test_no_extension", - size = "small", - a = ":fixture_no_extension.txt", - b = ":execute_no_extension", +# We expect the url passed to curl to just be the url with a trailing +# slash, without a filename. That way curl will use the original file. +curl_upload_file( + name = "upload_original_filename", + testonly = True, + src = ":file_to_upload", + url = "https://test.case/directory/", ) + +[ + ( + genrule( + name = "execute_{}".format(test), + testonly = True, + outs = [ + "upload_{}.out".format(test), + ], + cmd = "./$(location :upload_{}) > $@".format(test), + tools = [":upload_{}".format(test)], + ), + diff_file_test( + name = "test_{}".format(test), + size = "small", + a = ":expected_{}.txt".format(test), + b = ":execute_{}".format(test), + ), + ) + for test in ["basic", "original_filename", "injection", "no_extension"] +] diff --git a/e2e/upload/file/fixture.txt b/e2e/upload/file/expected_basic.txt similarity index 87% rename from e2e/upload/file/fixture.txt rename to e2e/upload/file/expected_basic.txt index 52ca57bb857d27c9ed34e84f722a09ce4481a253..db6b94c1344d67f86a4a42f8e7ae906e4bb5c740 100644 --- a/e2e/upload/file/fixture.txt +++ b/e2e/upload/file/expected_basic.txt @@ -6,5 +6,6 @@ --retry-delay 1 --upload-file -fixture.txt +input.txt +--url https://test.case/directory/fixture.txt diff --git a/e2e/upload/file/expected_injection.txt b/e2e/upload/file/expected_injection.txt new file mode 100644 index 0000000000000000000000000000000000000000..f4f08ab471bbd965982068fc860ffb799dfdf573 --- /dev/null +++ b/e2e/upload/file/expected_injection.txt @@ -0,0 +1,11 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +input.txt +--url +https://test.case/directory/injection/fixture.txt diff --git a/e2e/upload/file/fixture_no_extension.txt b/e2e/upload/file/expected_no_extension.txt similarity index 90% rename from e2e/upload/file/fixture_no_extension.txt rename to e2e/upload/file/expected_no_extension.txt index 806c14a07c5e5e2acd3136f902e6b61100c78b1b..c503ff42132c603db9950cb366e9c5b3b4224ee9 100644 --- a/e2e/upload/file/fixture_no_extension.txt +++ b/e2e/upload/file/expected_no_extension.txt @@ -6,5 +6,6 @@ --retry-delay 1 --upload-file -fixture.txt +input +--url https://test.case/directory/fixture diff --git a/e2e/upload/file/expected_original_filename.txt b/e2e/upload/file/expected_original_filename.txt new file mode 100644 index 0000000000000000000000000000000000000000..68cf35f257374def8337201f6d503ab271d0857d --- /dev/null +++ b/e2e/upload/file/expected_original_filename.txt @@ -0,0 +1,11 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +input.txt +--url +https://test.case/directory/ diff --git a/e2e/upload/manifests/BUILD.bazel b/e2e/upload/manifests/BUILD.bazel deleted file mode 100644 index b60e578b54c963a25c6f8a2c9bcdc03ad029afa7..0000000000000000000000000000000000000000 --- a/e2e/upload/manifests/BUILD.bazel +++ /dev/null @@ -1,71 +0,0 @@ -load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") -load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("//mock/manifest:rule.bzl", "mock_manifest") -load("//mock/manifests:rule.bzl", "mock_manifests") - -genrule( - name = "data", - testonly = True, - outs = [ - "data.out", - ], - cmd = "echo 'Hello data\n' > $@", -) - -mock_manifest( - name = "fixture_1", -) - -mock_manifest( - name = "fixture_2", -) - -mock_manifests( - name = "file_pack_1", - testonly = True, - outs = [ - "file_1.out", - "file_2.out", - ], -) - -mock_manifests( - name = "file_pack_2", - testonly = True, - outs = [ - "file_A.txt", - "file_B.txt", - ], - deps = [ - ":data", - ":file_pack_1", - ":fixture_2", - ], -) - -curl_upload_manifests( - name = "upload_files", - testonly = True, - srcs = [ - ":file_pack_2", - ":fixture_1", - ], - url = "https://test.case", -) - -genrule( - name = "execute", - testonly = True, - outs = [ - "upload_files.out", - ], - cmd = "./$(location :upload_files) > $@", - tools = [":upload_files"], -) - -diff_file_test( - name = "test", - size = "small", - a = ":fixture.txt", - b = ":execute", -) diff --git a/e2e/upload/manifests/fixture.txt b/e2e/upload/manifests/fixture.txt deleted file mode 100644 index 5dbf8f747158fedceb3994c2bb9560b70a233004..0000000000000000000000000000000000000000 --- a/e2e/upload/manifests/fixture.txt +++ /dev/null @@ -1,60 +0,0 @@ ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_1.out -https://test.case/upload/manifests/file_1.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_2.out -https://test.case/upload/manifests/file_2.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_A.txt -https://test.case/upload/manifests/file_A.txt ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_B.txt -https://test.case/upload/manifests/file_B.txt ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -fixture_2.out -https://test.case/upload/manifests/fixture_2.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -fixture_1.out -https://test.case/upload/manifests/fixture_1.out