From 91ec08e0f73f2138367b73e7bd3ca81a313e1291 Mon Sep 17 00:00:00 2001 From: Luke Hackwell Date: Tue, 14 Jan 2025 15:23:15 +0000 Subject: [PATCH 1/4] refactor: Remove POSIX/NT launcher scripts --- MODULE.bazel | 3 + curl/template/BUILD.bazel | 6 +- curl/template/template.go | 51 ++--------- curl/upload/BUILD.bazel | 27 +++--- curl/upload/file/rule.bzl | 59 ++++++------- curl/upload/manifests/rule.bzl | 52 ++++++------ curl/upload/nt.tmpl.bat | 4 - curl/upload/posix.tmpl.sh | 63 -------------- curl/upload/upload.go | 151 +++++++++++++++++++++++++++++++++ 9 files changed, 233 insertions(+), 183 deletions(-) delete mode 100644 curl/upload/nt.tmpl.bat delete mode 100644 curl/upload/posix.tmpl.sh create mode 100644 curl/upload/upload.go diff --git a/MODULE.bazel b/MODULE.bazel index 8bf7bb2..bfb3e2e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,6 +10,9 @@ bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.18") bazel_dep(name = "ape", version = "1.0.0-beta.17") bazel_dep(name = "rules_go", version = "0.48.1") +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.23.4") + export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export") use_repo(export, "ape-curl") export.symlink( diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel index d22934d..b27161b 100644 --- a/curl/template/BUILD.bazel +++ b/curl/template/BUILD.bazel @@ -1,8 +1,8 @@ -load("@rules_go//go:def.bzl", "go_binary") +load("@rules_go//go:def.bzl", "go_library") -go_binary( +go_library( name = "template", srcs = ["template.go"], - pure = "on", + importpath = "gitlab.arm.com/bazel/rules_curl/curl/template", visibility = ["//curl/upload:__subpackages__"], ) diff --git a/curl/template/template.go b/curl/template/template.go index 1a0d740..5cb4062 100644 --- a/curl/template/template.go +++ b/curl/template/template.go @@ -1,9 +1,7 @@ -package main +package template import ( "bytes" - "flag" - "fmt" "html/template" "log" "net/url" @@ -12,11 +10,6 @@ import ( "strings" ) -var manifest ManifestInfo = ManifestInfo{ - URL: URLInfo{}, - File: FileInfo{}, -} - type StringInfo string func (s StringInfo) String() string { @@ -26,12 +19,7 @@ func (s StringInfo) String() string { func (s *StringInfo) Set(value string) error { buf := new(bytes.Buffer) - tmpl, err := template.New("templ").Parse(value) - if err != nil { - log.Fatal(err) - } - - err = tmpl.Execute(buf, manifest) + _, err := template.New("templ").Parse(value) if err != nil { log.Fatal(err) } @@ -164,35 +152,8 @@ type ManifestInfo struct { File FileInfo } -func main() { - flag.Var(&manifest.File, "file", "The file path to use for templating") - flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file") - flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file") - flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file") - flag.Var(&manifest.URL, "url", "URL to use for templating") - flag.Var(&manifest.URL.Host, "host", "The domain name of the URL") - flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL") - flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error { - return manifest.URL.Pathname.Set(s) - }) - flag.Func("origin", "The origin of the represented URL.", func(s string) error { - index := strings.Index(s, "//") - - if index == -1 { - manifest.URL.Protocol = StringInfo(s) - return nil - } - - manifest.URL.Host = StringInfo(s[index+2:]) - manifest.URL.Protocol = StringInfo(s[:index]) - - return nil - }) - t := flag.String("template", "{{.URL.Href}}/{{.File.Path}}", "The Go template to render") - - flag.Parse() - - tmpl, err := template.New("url").Parse(*t) +func Tmpl(manifest ManifestInfo, template_s string) string { + tmpl, err := template.New("url").Parse(template_s) if err != nil { log.Fatal(err) } @@ -208,6 +169,6 @@ func main() { if err != nil { log.Fatal(err) } - - fmt.Printf("%s", buf.String()) + + return buf.String() } diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 04169ed..9398349 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -1,22 +1,19 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) +load("@rules_go//go:def.bzl", "go_binary") -alias( - name = "script", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_manifests`", - ), - visibility = ["//curl/upload:__subpackages__"], -) sh_binary( name = "csv", srcs = ["csv.sh"], visibility = ["//curl/upload:__subpackages__"], ) + +go_binary( + name = "upload", + srcs = ["upload.go"], + visibility = ["//curl/upload:__subpackages__"], + deps = [ + "//curl/template", + "@rules_go//go/runfiles:go_default_library", + ], + pure = "on" +) diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index ddad0e1..030adf1 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -36,16 +36,11 @@ ATTRS = { doc = "The seconds to wait before attempting a upload retry.", default = 1, ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", + "_upload": attr.label( + default = "//curl/upload", allow_single_file = True, executable = True, + cfg = "exec" ), "_csv": attr.label( doc = "CSV tool", @@ -55,6 +50,13 @@ ATTRS = { ), } + +def _runfile(label, file): + path = file.short_path + if path.startswith("../"): + return path.removeprefix("../") + return "{}/{}".format(label.workspace_name or "_main", path) + def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] @@ -63,7 +65,9 @@ def implementation(ctx): args = ctx.actions.args() # SRC, DST, TEMPLATE, URL - args.add("{},{},{},{}".format(ctx.file.src.short_path, ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) + args.add("{},{},{},{}".format( + _runfile(ctx.file.src.owner, ctx.file.src), + ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) ctx.actions.run( outputs = [csv], @@ -72,31 +76,28 @@ def implementation(ctx): executable = ctx.executable._csv, mnemonic = "PrepareUploadCSV", ) - - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - substitutions.add("{{directory}}", str(ctx.file.src.short_path)) - - ctx.actions.expand_template( - template = ctx.file._script, + arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) + args = ctx.actions.args() + + args.add("curl", _runfile(curl.executable.owner, curl.executable)) + args.add("csv", _runfile(csv.owner, csv)) + args.add("retry", str(ctx.attr.retry)) + args.add("retry-delay", str(ctx.attr.retry_delay)) + ctx.actions.write(output = arguments, content = args) + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( output = executable, - computed_substitutions = substitutions, + target_file = ctx.executable._upload, is_executable = True, ) - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, ctx.file.src, csv]) - runfiles = runfiles.merge(ctx.attr.src.default_runfiles) + files = depset([executable, arguments]) + root_symlinks = { "upload.args": arguments } + + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv, ctx.file.src], root_symlinks = root_symlinks) runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) - runfiles = runfiles.merge(ctx.attr._csv.default_runfiles) - + return DefaultInfo( executable = executable, files = files, diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 713ebd9..74b6bf1 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -40,16 +40,11 @@ ATTRS = { doc = "The seconds to wait before attempting an upload retry.", default = 1, ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", + "_upload": attr.label( + default = "//curl/upload", allow_single_file = True, executable = True, + cfg = "exec", ), "_csv": attr.label( doc = "CSV tool", @@ -59,6 +54,12 @@ ATTRS = { ), } +def _runfile(label, file): + path = file.short_path + if path.startswith("../"): + return path.removeprefix("../") + return "{}/{}".format(label.workspace_name or "_main", path) + def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] @@ -72,7 +73,8 @@ def implementation(ctx): def _to_string(m): # SRC, DST, TEMPLATE, URL - return "{},{},{},{}".format(m.file.short_path, m.file.short_path, m.url, href) + return "{},{},{},{}".format( + _runfile(m.file.owner, m.file), m.file.short_path, m.url, href) args = ctx.actions.args() args.add_all(manifests, map_each = _to_string, allow_closure = True) @@ -84,26 +86,28 @@ def implementation(ctx): mnemonic = "PrepareUploadCSV", ) - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - - ctx.actions.expand_template( - template = ctx.file._script, + arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) + args = ctx.actions.args() + args.add("curl", _runfile(curl.executable.owner, curl.executable)) + args.add("csv", _runfile(csv.owner, csv)) + args.add("retry", str(ctx.attr.retry)) + args.add("retry-delay", str(ctx.attr.retry_delay)) + ctx.actions.write(output = arguments, content = args) + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( output = executable, - computed_substitutions = substitutions, + target_file = ctx.executable._upload, is_executable = True, ) - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, csv, ctx.executable._template] + [m.file for m in manifests.to_list()]) + files = depset([executable, arguments]) + root_symlinks = { "upload.args": arguments } + + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv], root_symlinks = root_symlinks) runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) + for manifest in manifests.to_list(): + runfiles = runfiles.merge(ctx.runfiles([manifest.file])) return DefaultInfo( executable = executable, diff --git a/curl/upload/nt.tmpl.bat b/curl/upload/nt.tmpl.bat deleted file mode 100644 index d522214..0000000 --- a/curl/upload/nt.tmpl.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -# TODO: implement Windows Batch for `curl_upload` -exit /b 121 diff --git a/curl/upload/posix.tmpl.sh b/curl/upload/posix.tmpl.sh deleted file mode 100644 index 05687f3..0000000 --- a/curl/upload/posix.tmpl.sh +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env sh - -# Strict shell -set -o errexit -o nounset - -# Runfiles location -rlocation() ( - readonly FILEPATH="${1}" - - if test -e "${FILEPATH}"; then - printf '%s' "${FILEPATH}" - return - fi - - readonly RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" - - if test -e "${RUNFILES_DIR}/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/${FILEPATH#../}" - return - fi - - if test -e "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/"*"/${FILEPATH#../}" - return - fi - - printf >&2 "No runfile found: %s\n" "${FILEPATH}" - exit 1 -) - -# Bazel substitutions -CURL="$(rlocation "{{curl}}")" -CSV="$(rlocation "{{csv}}")" -TEMPLATE="$(rlocation "{{template}}")" -RETRY="{{retry}}" -RETRY_DELAY="{{retry_delay}}" -readonly CURL CSV TEMPLATE RETRY RETRY_DELAY - -# Uploads a file -upload() { - printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" - - # Do the upload - "${CURL}" \ - --netrc \ - --location \ - --progress-bar \ - --retry "${RETRY}" \ - --retry-delay "${RETRY_DELAY}" \ - --upload-file "$(rlocation "${1}")" \ - "${2}" -} - -while IFS=, read -r SRC DST TMPL URL; do - FULL_DST=$( - "${TEMPLATE}" \ - --url "${URL}" \ - --template "${TMPL}" \ - --file "${DST}" \ - ${@} - ) - upload "${SRC}" "${FULL_DST}" -done <"${CSV}" diff --git a/curl/upload/upload.go b/curl/upload/upload.go new file mode 100644 index 0000000..8985d15 --- /dev/null +++ b/curl/upload/upload.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/csv" + "flag" + "io" + "iter" + "os" + "log" + "os/exec" + "strconv" + "strings" + "gitlab.arm.com/bazel/rules_curl/curl/template" + "github.com/bazelbuild/rules_go/go/runfiles" +) + +type RunfileVar string + +func (r RunfileVar) String() string { + return string(r) +} + +func (r *RunfileVar) Set(value string) error { + runfile, err := runfiles.Rlocation(value) + if err != nil { log.Fatal(err) } + + *r = RunfileVar(runfile) + return nil +} + +type ManifestsVar struct { + reader *csv.Reader + path string +} + +func (m ManifestsVar) String() string { + return m.path +} + +func (m ManifestsVar) Iterate() iter.Seq2[[2]string, error] { + return func(yield func([2]string, error) bool) { + var empty [2]string + for { + record, err := m.reader.Read() + if err == io.EOF { + return + } + if err != nil { + if !yield(empty, err) { + return + } + } + + var manifest template.ManifestInfo = template.ManifestInfo { + URL: template.URLInfo{}, + File: template.FileInfo{}, + } + + manifest.URL.Set(record[3]) + manifest.File.Set(record[1]) + url := template.Tmpl(manifest, record[2]) + + item := [2]string{record[0], url} + + if !yield(item, nil) { + return + } + } + } +} + +func (m *ManifestsVar) Set(value string) error { + csv_path, err := runfiles.Rlocation(value) + if err != nil { log.Fatal(err) } + + m.path = csv_path + + csv_byt, err := os.ReadFile(csv_path) + if err != nil { log.Fatal(err) } + + m.reader = csv.NewReader(strings.NewReader(string(csv_byt))) + return nil +} + +func upload(curl RunfileVar, src string, dest string, retry uint64, retry_delay uint64) error { + args := []string{ + "--netrc", + "--location", + "--progress-bar", + "--retry", strconv.FormatUint(retry, 10), + "--retry-delay", strconv.FormatUint(retry_delay, 10), + "--upload-file", src, + dest, + } + cmd := exec.Command(curl.String(), args...) + + cmd.Stdout = os.Stdout + + err := cmd.Run() + if err != nil { return err } + + return nil +} + +func setArgs(args_file string) error { + args_byt, err := os.ReadFile(args_file) + if err != nil { return err } + + args := strings.Split(string(args_byt), "\n") + // Remove ending newline + args = args[:len(args)-1] + + for i, _ := range args { + if i%2 != 0 { + err = flag.Set(args[i-1], args[i]) + if err != nil { return err } + } + } + + return nil +} + +func main() { + var curl RunfileVar + var manifests ManifestsVar + var retry uint64 + var retry_delay uint64 + + flag.Var(&curl, "curl", "The path to the curl bin") + flag.Var(&manifests, "csv", "The path to the csv") + flag.Uint64Var(&retry, "retry", 0, "The number of times to retry the request") + flag.Uint64Var(&retry_delay, "retry-delay", 1, "The number of seconds to wait before retrying the request") + + args_file, err := runfiles.Rlocation("upload.args") + if err != nil { log.Fatal(err) } + + err = setArgs(args_file) + if err != nil { log.Fatal(err) } + + flag.Parse() + + for upload_info, err := range manifests.Iterate() { + if err != nil { log.Fatal(err) } + + src, err := runfiles.Rlocation(upload_info[0]) + if err != nil { log.Fatal(err) } + + err = upload(curl, src, upload_info[1], retry, retry_delay) + if err != nil { log.Fatal(err) } + } +} -- GitLab From a9bc07824e1dd2f659395b65ac206f59c627e2cd Mon Sep 17 00:00:00 2001 From: Jonathan Watson Date: Thu, 23 Jan 2025 17:09:53 +0000 Subject: [PATCH 2/4] feat: remove curl_upload_manifest BREAKING CHANGE: curl_upload_manifest removed --- README.md | 38 --------- curl/upload/ManifestInfo.bzl | 58 -------------- curl/upload/ManifestsInfo.bzl | 34 -------- curl/upload/file/rule.bzl | 10 +-- curl/upload/manifest/BUILD.bazel | 0 curl/upload/manifest/defs.bzl | 5 -- curl/upload/manifests/BUILD.bazel | 0 curl/upload/manifests/defs.bzl | 9 --- curl/upload/manifests/rule.bzl | 126 ------------------------------ e2e/mock/manifest/BUILD.bazel | 0 e2e/mock/manifest/rule.bzl | 33 -------- e2e/mock/manifests/BUILD.bazel | 0 e2e/mock/manifests/rule.bzl | 56 ------------- e2e/upload/manifests/BUILD.bazel | 71 ----------------- e2e/upload/manifests/fixture.txt | 60 -------------- 15 files changed, 4 insertions(+), 496 deletions(-) delete mode 100644 curl/upload/ManifestInfo.bzl delete mode 100644 curl/upload/ManifestsInfo.bzl delete mode 100644 curl/upload/manifest/BUILD.bazel delete mode 100644 curl/upload/manifest/defs.bzl delete mode 100644 curl/upload/manifests/BUILD.bazel delete mode 100644 curl/upload/manifests/defs.bzl delete mode 100644 curl/upload/manifests/rule.bzl delete mode 100644 e2e/mock/manifest/BUILD.bazel delete mode 100644 e2e/mock/manifest/rule.bzl delete mode 100644 e2e/mock/manifests/BUILD.bazel delete mode 100644 e2e/mock/manifests/rule.bzl delete mode 100644 e2e/upload/manifests/BUILD.bazel delete mode 100644 e2e/upload/manifests/fixture.txt diff --git a/README.md b/README.md index d88cecd..e7c9f5b 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,6 @@ Add the following to `MODULE.bazel`: bazel_dep(module_name = "rules_curl", version = "0.0.0") ``` -## Providers - -### ManifestInfo - -Encapsulates the file to upload and its URL template (see [ManifestInfo][manifest_info]) - -Members: - -- **file**: File -- **url**: string - -### ManifestsInfo - -The dependency set of `ManifestInfo`s - -Members: - -- **manifests**: depset of `ManifestInfo`s - ## Usage ### curl_upload_file @@ -44,22 +25,3 @@ curl_upload_file( ) ``` - -### curl_upload_manifests - -```py -load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") - -curl_upload_manifests( - name = "upload", - srcs = [ - ":fixture_1", - ":fixture_2", - ], - url = "https://test.case", -) -``` - -> Note: `fixture_1` and `fixture_2` have to provide either `ManifestInfo` or `ManifestsInfo` - -[manifest_info]: curl/upload/ManifestInfo.bzl diff --git a/curl/upload/ManifestInfo.bzl b/curl/upload/ManifestInfo.bzl deleted file mode 100644 index 2694446..0000000 --- a/curl/upload/ManifestInfo.bzl +++ /dev/null @@ -1,58 +0,0 @@ -load("@bazel_skylib//lib:types.bzl", "types") - -visibility("//...") - -def init(file, url): - """ - Initializes a `CurlUploadManifestInfo` provider. - - To be used with `curl_upload_manifests` - - Args: - file: The file to upload. - url: A Starlark string template for the URL, which can include: - - {{.URL.Href}}: the provided URL - - {{.URL.Auth}}: the user:password auth - - {{.URL.Origin}}: the origin of the represented URL - - {{.URL.Port}}: the port number of the URL - - {{.URL.Host}}: the hostname followed by the port if not empty - - {{.URL.Hostname}}: the domain name of the URL - - {{.URL.Protocol}}: the protocol scheme of the URL - - {{.URL.Pathname}}: a location in a hierarchical structure - - {{.File.Path}}: the full path of the file - - {{.File.Basename}}: the basename of the file - - {{.File.Dirname}}: the directory of the file - - {{.File.Stem}}: the file name without extension - - {{.File.Extension}}: the extension of the file - - Returns: - A mapping of keywords for the `curl_upload_manifest_info` raw constructor. - """ - if type(file) != "File": - fail("`CurlUploadManifestInfo.file` must be a `file`: {}".format(file)) - - if file.is_directory: - fail("`CurlUploadManifestInfo.file` must not be a directory: {}".format(file)) - - if not types.is_string(url): - fail("`CurlUploadManifestInfo.url` must be a `str`: {}".format(url)) - - if url.find(",") != -1: - fail("`CurlUploadManifestInfo.url` must not have comma `,` sign: {}".format(url)) - - return { - "file": file, - "url": url, - } - -CurlUploadManifestInfo, curl_upload_manifest_info = provider( - "A file to upload with cURL.", - fields = ["file", "url"], - init = init, -) - -# Provide some convenience imports -ManifestInfo = CurlUploadManifestInfo -manifest_info = curl_upload_manifest_info -Info = CurlUploadManifestInfo -info = curl_upload_manifest_info diff --git a/curl/upload/ManifestsInfo.bzl b/curl/upload/ManifestsInfo.bzl deleted file mode 100644 index 800e6d3..0000000 --- a/curl/upload/ManifestsInfo.bzl +++ /dev/null @@ -1,34 +0,0 @@ -load("@bazel_skylib//lib:types.bzl", "types") - -visibility("//...") - -def init(manifests): - """ - Initializes a `CurlUploadManifestsInfo` provider. - - To be used with `curl_upload_manifests` - - Args: - manifests: The dependency set of `CurlUploadManifestInfo`s - - Returns: - A mapping of keywords for the `curl_upload_manifests_info` raw constructor. - """ - if not types.is_depset(manifests): - fail("`CurlUploadManifestsInfo.url` must be a `depset`: {}".format(manifests)) - - return { - "manifests": manifests, - } - -CurlUploadManifestsInfo, curl_upload_manifests_info = provider( - "Files to upload with cURL.", - fields = ["manifests"], - init = init, -) - -# Provide some convenience imports -ManifestsInfo = CurlUploadManifestsInfo -manifests_info = curl_upload_manifests_info -Info = CurlUploadManifestsInfo -info = curl_upload_manifests_info diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index 030adf1..67bff61 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -1,5 +1,3 @@ -load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info") - visibility("//curl/...") DOC = """Upload a file to a URL endpoint with cURL. @@ -78,13 +76,13 @@ def implementation(ctx): ) arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) args = ctx.actions.args() - + args.add("curl", _runfile(curl.executable.owner, curl.executable)) args.add("csv", _runfile(csv.owner, csv)) args.add("retry", str(ctx.attr.retry)) args.add("retry-delay", str(ctx.attr.retry_delay)) ctx.actions.write(output = arguments, content = args) - + executable = ctx.actions.declare_file(ctx.label.name) ctx.actions.symlink( output = executable, @@ -94,10 +92,10 @@ def implementation(ctx): files = depset([executable, arguments]) root_symlinks = { "upload.args": arguments } - + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv, ctx.file.src], root_symlinks = root_symlinks) runfiles = runfiles.merge(curl.default.default_runfiles) - + return DefaultInfo( executable = executable, files = files, diff --git a/curl/upload/manifest/BUILD.bazel b/curl/upload/manifest/BUILD.bazel deleted file mode 100644 index e69de29..0000000 diff --git a/curl/upload/manifest/defs.bzl b/curl/upload/manifest/defs.bzl deleted file mode 100644 index ce7ef89..0000000 --- a/curl/upload/manifest/defs.bzl +++ /dev/null @@ -1,5 +0,0 @@ -load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info") - -visibility("public") - -CurlUploadManifestInfo = _Manifest diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel deleted file mode 100644 index e69de29..0000000 diff --git a/curl/upload/manifests/defs.bzl b/curl/upload/manifests/defs.bzl deleted file mode 100644 index 0da0eba..0000000 --- a/curl/upload/manifests/defs.bzl +++ /dev/null @@ -1,9 +0,0 @@ -load(":rule.bzl", _manifests = "manifests") -load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info") -load("//curl/upload:ManifestsInfo.bzl", _Manifests = "Info") - -visibility("public") - -curl_upload_manifests = _manifests -CurlUploadManifestInfo = _Manifest -CurlUploadManifestsInfo = _Manifests diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl deleted file mode 100644 index 74b6bf1..0000000 --- a/curl/upload/manifests/rule.bzl +++ /dev/null @@ -1,126 +0,0 @@ -load("//curl/upload:ManifestInfo.bzl", "ManifestInfo") -load("//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") - -visibility("//curl/...") - -DOC = """Upload bunch of files to a URL endpoint with cURL. - -The `srcs` must provide `ManifestInfo` or `ManifestsInfo`. - -```py -file( - name = "upload_files", - srcs = [ - ":data", - ], - url = "https://host.name.to.upload", -) -``` -""" - -ATTRS = { - "srcs": attr.label_list( - doc = "Files to be uploaded.", - mandatory = True, - providers = [ - [ManifestInfo], - [ManifestsInfo], - ], - allow_files = False, - ), - "url": attr.string( - doc = "URL endpoint for files to upload.", - mandatory = True, - ), - "retry": attr.int( - doc = "The number of retry attempts for every file.", - default = 3, - ), - "retry_delay": attr.int( - doc = "The seconds to wait before attempting an upload retry.", - default = 1, - ), - "_upload": attr.label( - default = "//curl/upload", - allow_single_file = True, - executable = True, - cfg = "exec", - ), - "_csv": attr.label( - doc = "CSV tool", - default = "//curl/upload:csv", - cfg = "exec", - executable = True, - ), -} - -def _runfile(label, file): - path = file.short_path - if path.startswith("../"): - return path.removeprefix("../") - return "{}/{}".format(label.workspace_name or "_main", path) - -def implementation(ctx): - curl = ctx.toolchains["//curl/toolchain/curl:type"] - - csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) - manifests = depset( - direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], - transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], - ) - - href = ctx.attr.url.rstrip("/").replace(",", "%2C") - - def _to_string(m): - # SRC, DST, TEMPLATE, URL - return "{},{},{},{}".format( - _runfile(m.file.owner, m.file), m.file.short_path, m.url, href) - - args = ctx.actions.args() - args.add_all(manifests, map_each = _to_string, allow_closure = True) - - ctx.actions.run( - outputs = [csv], - arguments = [csv.path, args], - executable = ctx.executable._csv, - mnemonic = "PrepareUploadCSV", - ) - - arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) - args = ctx.actions.args() - args.add("curl", _runfile(curl.executable.owner, curl.executable)) - args.add("csv", _runfile(csv.owner, csv)) - args.add("retry", str(ctx.attr.retry)) - args.add("retry-delay", str(ctx.attr.retry_delay)) - ctx.actions.write(output = arguments, content = args) - - executable = ctx.actions.declare_file(ctx.label.name) - ctx.actions.symlink( - output = executable, - target_file = ctx.executable._upload, - is_executable = True, - ) - - files = depset([executable, arguments]) - root_symlinks = { "upload.args": arguments } - - runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv], root_symlinks = root_symlinks) - runfiles = runfiles.merge(curl.default.default_runfiles) - for manifest in manifests.to_list(): - runfiles = runfiles.merge(ctx.runfiles([manifest.file])) - - return DefaultInfo( - executable = executable, - files = files, - runfiles = runfiles, - ) - -curl_upload_manifests = rule( - doc = DOC, - attrs = ATTRS, - implementation = implementation, - toolchains = ["//curl/toolchain/curl:type"], - executable = True, -) - -manifests = curl_upload_manifests diff --git a/e2e/mock/manifest/BUILD.bazel b/e2e/mock/manifest/BUILD.bazel deleted file mode 100644 index e69de29..0000000 diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl deleted file mode 100644 index 8dcd738..0000000 --- a/e2e/mock/manifest/rule.bzl +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_curl//curl/upload/manifest:defs.bzl", ManifestInfo = "CurlUploadManifestInfo") - -DOC = """Mock rule with output `ManifestInfo` provider. - -```py -mock_manifest( - name = "mock_file", -) -``` -""" - -def _impl(ctx): - out = ctx.actions.declare_file("{}.out".format(ctx.attr.name)) - - ctx.actions.write(out, "hello test", is_executable = False) - - return [ - DefaultInfo( - files = depset([out]), - ), - ManifestInfo( - file = out, - url = "{{.URL.Href}}/{{.File.Path}}", - ), - ] - -mock_manifest = rule( - doc = DOC, - implementation = _impl, - provides = [ - ManifestInfo, - ], -) diff --git a/e2e/mock/manifests/BUILD.bazel b/e2e/mock/manifests/BUILD.bazel deleted file mode 100644 index e69de29..0000000 diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl deleted file mode 100644 index d786e4e..0000000 --- a/e2e/mock/manifests/rule.bzl +++ /dev/null @@ -1,56 +0,0 @@ -load("@rules_curl//curl/upload/manifests:defs.bzl", ManifestsInfo = "CurlUploadManifestsInfo", ManifestInfo = "CurlUploadManifestInfo") - -DOC = """Mock rule with output `ManifestsInfo` provider. - -```py -mock_manifests( - name = "mock_files", - outs = [ - "file_1.out", - "file_2.out", - ], - deps = [ - "my_dep", - ], -) -``` -""" - -ATTRS = { - "outs": attr.string_list( - default = [], - mandatory = True, - ), - "deps": attr.label_list(allow_files = True), -} - -def _impl(ctx): - outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] - - for f in outs: - ctx.actions.write(f, "hello test", is_executable = False) - - files = depset(outs) - - manifests = [ManifestInfo(file = o, url = "{{.URL.Href}}/{{.File.Path}}") for o in outs] - manifests.extend([dep[ManifestInfo] for dep in ctx.attr.deps if ManifestInfo in dep]) - - transitive = [dep[ManifestsInfo].manifests for dep in ctx.attr.deps if ManifestsInfo in dep] - - return [ - DefaultInfo( - files = files, - ), - ManifestsInfo( - manifests = depset(direct = manifests, transitive = transitive), - ), - ] - -mock_manifests = rule( - doc = DOC, - attrs = ATTRS, - implementation = _impl, - provides = [ - ManifestsInfo, - ], -) diff --git a/e2e/upload/manifests/BUILD.bazel b/e2e/upload/manifests/BUILD.bazel deleted file mode 100644 index b60e578..0000000 --- a/e2e/upload/manifests/BUILD.bazel +++ /dev/null @@ -1,71 +0,0 @@ -load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") -load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("//mock/manifest:rule.bzl", "mock_manifest") -load("//mock/manifests:rule.bzl", "mock_manifests") - -genrule( - name = "data", - testonly = True, - outs = [ - "data.out", - ], - cmd = "echo 'Hello data\n' > $@", -) - -mock_manifest( - name = "fixture_1", -) - -mock_manifest( - name = "fixture_2", -) - -mock_manifests( - name = "file_pack_1", - testonly = True, - outs = [ - "file_1.out", - "file_2.out", - ], -) - -mock_manifests( - name = "file_pack_2", - testonly = True, - outs = [ - "file_A.txt", - "file_B.txt", - ], - deps = [ - ":data", - ":file_pack_1", - ":fixture_2", - ], -) - -curl_upload_manifests( - name = "upload_files", - testonly = True, - srcs = [ - ":file_pack_2", - ":fixture_1", - ], - url = "https://test.case", -) - -genrule( - name = "execute", - testonly = True, - outs = [ - "upload_files.out", - ], - cmd = "./$(location :upload_files) > $@", - tools = [":upload_files"], -) - -diff_file_test( - name = "test", - size = "small", - a = ":fixture.txt", - b = ":execute", -) diff --git a/e2e/upload/manifests/fixture.txt b/e2e/upload/manifests/fixture.txt deleted file mode 100644 index 5dbf8f7..0000000 --- a/e2e/upload/manifests/fixture.txt +++ /dev/null @@ -1,60 +0,0 @@ ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_1.out -https://test.case/upload/manifests/file_1.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_2.out -https://test.case/upload/manifests/file_2.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_A.txt -https://test.case/upload/manifests/file_A.txt ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -file_B.txt -https://test.case/upload/manifests/file_B.txt ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -fixture_2.out -https://test.case/upload/manifests/fixture_2.out ---netrc ---location ---progress-bar ---retry -3 ---retry-delay -1 ---upload-file -fixture_1.out -https://test.case/upload/manifests/fixture_1.out -- GitLab From 729b41d7a3d51cb96c25efd3557bff7ecc10bf3e Mon Sep 17 00:00:00 2001 From: Jonathan Watson Date: Thu, 23 Jan 2025 17:04:06 +0000 Subject: [PATCH 3/4] feat: allow make variable substitution of the URL BREAKING CHANGE: removes `dst` attribute --- MODULE.bazel | 6 + MODULE.bazel.lock | 6 + curl/template/BUILD.bazel | 8 - curl/template/template.go | 174 ------------------ curl/upload/BUILD.bazel | 12 +- curl/upload/csv.sh | 13 -- curl/upload/file/rule.bzl | 44 ++--- curl/upload/go.mod | 8 + curl/upload/go.sum | 2 + curl/upload/upload.go | 124 ++++--------- e2e/MODULE.bazel.lock | 6 + e2e/upload/file/BUILD.bazel | 89 ++++++++- e2e/upload/file/fixture.txt | 3 +- e2e/upload/file/fixture_injection.txt | 11 ++ e2e/upload/file/fixture_no_extension.txt | 3 +- e2e/upload/file/fixture_original_filename.txt | 11 ++ 16 files changed, 193 insertions(+), 327 deletions(-) delete mode 100644 curl/template/BUILD.bazel delete mode 100644 curl/template/template.go delete mode 100755 curl/upload/csv.sh create mode 100644 curl/upload/go.mod create mode 100644 curl/upload/go.sum create mode 100644 e2e/upload/file/fixture_injection.txt create mode 100644 e2e/upload/file/fixture_original_filename.txt diff --git a/MODULE.bazel b/MODULE.bazel index bfb3e2e..4a87144 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,12 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.18") bazel_dep(name = "ape", version = "1.0.0-beta.17") bazel_dep(name = "rules_go", version = "0.48.1") +bazel_dep(name = "gazelle", version = "0.31.0") + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//curl/upload:go.mod") +use_repo( + go_deps, "com_github_google_shlex") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.23.4") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a7d227e..5853721 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -28,6 +28,9 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.31.0/MODULE.bazel": "0319690246f72d0b5d596724a0ea0da2fd823905643a042c95bc2c420438ddae", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", @@ -52,6 +55,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel deleted file mode 100644 index b27161b..0000000 --- a/curl/template/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "template", - srcs = ["template.go"], - importpath = "gitlab.arm.com/bazel/rules_curl/curl/template", - visibility = ["//curl/upload:__subpackages__"], -) diff --git a/curl/template/template.go b/curl/template/template.go deleted file mode 100644 index 5cb4062..0000000 --- a/curl/template/template.go +++ /dev/null @@ -1,174 +0,0 @@ -package template - -import ( - "bytes" - "html/template" - "log" - "net/url" - "path/filepath" - "strconv" - "strings" -) - -type StringInfo string - -func (s StringInfo) String() string { - return string(s) -} - -func (s *StringInfo) Set(value string) error { - buf := new(bytes.Buffer) - - _, err := template.New("templ").Parse(value) - if err != nil { - log.Fatal(err) - } - - *s = StringInfo(buf.String()) - return nil -} - -type FileInfo struct { - Dirname StringInfo - Stem StringInfo - Extension StringInfo -} - -func (f FileInfo) Basename() string { - extension := f.Extension.String() - stem := f.Stem.String() - if extension == "" { - return stem - } - return stem + "." + extension -} - -func (f FileInfo) Path() string { - return f.Dirname.String() + "/" + f.Basename() -} - -func (f FileInfo) String() string { - return f.Path() -} - -func (f *FileInfo) Set(value string) error { - clean := filepath.Clean(value) - dirname := filepath.Dir(clean) - f.Dirname = StringInfo(dirname) - - extension := filepath.Ext(value) - nodot := strings.TrimPrefix(extension, ".") - f.Extension = StringInfo(nodot) - - base := filepath.Base(value) - stem := strings.TrimSuffix(base, extension) - f.Stem = StringInfo(stem) - return nil -} - -type URLInfo struct { - Username *StringInfo - Password *StringInfo - Host StringInfo - Pathname StringInfo - Protocol StringInfo -} - -func (u URLInfo) split() (string, *int) { - host := u.Host.String() - index := strings.LastIndexByte(host, ':') - - if index == -1 { - return host, nil - } - - hostname := host[:index] - port, err := strconv.Atoi(host[index+1:]) - if err != nil { - log.Fatal(err) - } - return hostname, &port -} - -func (u URLInfo) Hostname() string { - hostname, _ := u.split() - return hostname -} - -func (u URLInfo) Port() *int { - _, port := u.split() - return port -} - -func (u URLInfo) Origin() string { - return u.Protocol.String() + "//" + u.Host.String() -} - -func (u URLInfo) Auth() (auth string) { - if u.Username != nil { - auth += string(*u.Username) - } - if u.Password != nil { - auth += ":" + string(*u.Password) - } - return -} - -func (u URLInfo) Href() (href string) { - href += u.Protocol.String() + "//" - if auth := u.Auth(); auth != "" { - href += auth + "@" - } - href += u.Host.String() - href += u.Pathname.String() - return -} - -func (u URLInfo) String() string { - return u.Href() -} - -func (u *URLInfo) Set(value string) error { - parsed, err := url.Parse(value) - if err != nil { - return err - } - if username := parsed.User.Username(); username != "" { - info := StringInfo(username) - u.Username = &info - } - if password, set := parsed.User.Password(); set { - info := StringInfo(password) - u.Password = &info - } - u.Host = StringInfo(parsed.Host) - u.Pathname = StringInfo(parsed.Path) - u.Protocol = StringInfo(parsed.Scheme + ":") - return err -} - -type ManifestInfo struct { - URL URLInfo - File FileInfo -} - -func Tmpl(manifest ManifestInfo, template_s string) string { - tmpl, err := template.New("url").Parse(template_s) - if err != nil { - log.Fatal(err) - } - - buf := new(bytes.Buffer) - - err = tmpl.Execute(buf, manifest) - if err != nil { - log.Fatal(err) - } - - _, err = url.Parse(buf.String()) - if err != nil { - log.Fatal(err) - } - - return buf.String() -} diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 9398349..ed6df26 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -1,19 +1,17 @@ load("@rules_go//go:def.bzl", "go_binary") -sh_binary( - name = "csv", - srcs = ["csv.sh"], - visibility = ["//curl/upload:__subpackages__"], -) - go_binary( name = "upload", srcs = ["upload.go"], visibility = ["//curl/upload:__subpackages__"], deps = [ - "//curl/template", "@rules_go//go/runfiles:go_default_library", + "@com_github_google_shlex//:shlex" ], pure = "on" ) + +exports_files([ + "go.mod", +]) diff --git a/curl/upload/csv.sh b/curl/upload/csv.sh deleted file mode 100755 index 39301d0..0000000 --- a/curl/upload/csv.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env sh - -set -o errexit -o nounset - -OUT="${1}" -shift -readonly OUT - -for ARG in "${@}"; do - if test -n "${ARG}"; then - printf >>"${OUT}" '%s\n' "${ARG}" - fi -done diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index 67bff61..e36cd4c 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -6,7 +6,6 @@ DOC = """Upload a file to a URL endpoint with cURL. file( name = "upload_file", src = ":data", - dst = "data" url = "https://host.name.to.upload", ) ``` @@ -18,12 +17,8 @@ ATTRS = { mandatory = True, allow_single_file = True, ), - "dst": attr.string( - doc = "The filename to upload as.", - mandatory = True, - ), "url": attr.string( - doc = "URL endpoint for file upload.", + doc = "URL endpoint for file upload. Subject to 'Make variable' expansion.", mandatory = True, ), "retry": attr.int( @@ -38,17 +33,10 @@ ATTRS = { default = "//curl/upload", allow_single_file = True, executable = True, - cfg = "exec" - ), - "_csv": attr.label( - doc = "CSV tool", - default = "//curl/upload:csv", cfg = "exec", - executable = True, - ), + ), } - def _runfile(label, file): path = file.short_path if path.startswith("../"): @@ -58,27 +46,19 @@ def _runfile(label, file): def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] - csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) - href = ctx.attr.url.rstrip("/").replace(",", "%2C") + if ctx.file.src.is_directory: + fail("'src' must be a file not a directory.") - args = ctx.actions.args() - # SRC, DST, TEMPLATE, URL - args.add("{},{},{},{}".format( - _runfile(ctx.file.src.owner, ctx.file.src), - ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) + # Do the 'Make variable' expansion on the URL. + url = ctx.attr.url + for k, v in ctx.var.items(): + url = url.replace("$({})".format(k), v) - ctx.actions.run( - outputs = [csv], - inputs = [ctx.file.src], - arguments = [csv.path, args], - executable = ctx.executable._csv, - mnemonic = "PrepareUploadCSV", - ) arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) args = ctx.actions.args() - + args.add("src", _runfile(ctx.file.src.owner, ctx.file.src)) + args.add("url", url) args.add("curl", _runfile(curl.executable.owner, curl.executable)) - args.add("csv", _runfile(csv.owner, csv)) args.add("retry", str(ctx.attr.retry)) args.add("retry-delay", str(ctx.attr.retry_delay)) ctx.actions.write(output = arguments, content = args) @@ -91,9 +71,9 @@ def implementation(ctx): ) files = depset([executable, arguments]) - root_symlinks = { "upload.args": arguments } + root_symlinks = {"upload.args": arguments} - runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv, ctx.file.src], root_symlinks = root_symlinks) + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, ctx.file.src], root_symlinks = root_symlinks) runfiles = runfiles.merge(curl.default.default_runfiles) return DefaultInfo( diff --git a/curl/upload/go.mod b/curl/upload/go.mod new file mode 100644 index 0000000..8d08116 --- /dev/null +++ b/curl/upload/go.mod @@ -0,0 +1,8 @@ +module curl_runner + +go 1.19 + + +require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +) diff --git a/curl/upload/go.sum b/curl/upload/go.sum new file mode 100644 index 0000000..a14fb86 --- /dev/null +++ b/curl/upload/go.sum @@ -0,0 +1,2 @@ +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= diff --git a/curl/upload/upload.go b/curl/upload/upload.go index 8985d15..bf3dcce 100644 --- a/curl/upload/upload.go +++ b/curl/upload/upload.go @@ -1,17 +1,15 @@ package main import ( - "encoding/csv" "flag" - "io" - "iter" - "os" "log" + "os" "os/exec" "strconv" "strings" - "gitlab.arm.com/bazel/rules_curl/curl/template" + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/google/shlex" ) type RunfileVar string @@ -22,67 +20,15 @@ func (r RunfileVar) String() string { func (r *RunfileVar) Set(value string) error { runfile, err := runfiles.Rlocation(value) - if err != nil { log.Fatal(err) } - - *r = RunfileVar(runfile) - return nil -} - -type ManifestsVar struct { - reader *csv.Reader - path string -} - -func (m ManifestsVar) String() string { - return m.path -} - -func (m ManifestsVar) Iterate() iter.Seq2[[2]string, error] { - return func(yield func([2]string, error) bool) { - var empty [2]string - for { - record, err := m.reader.Read() - if err == io.EOF { - return - } - if err != nil { - if !yield(empty, err) { - return - } - } - - var manifest template.ManifestInfo = template.ManifestInfo { - URL: template.URLInfo{}, - File: template.FileInfo{}, - } - - manifest.URL.Set(record[3]) - manifest.File.Set(record[1]) - url := template.Tmpl(manifest, record[2]) - - item := [2]string{record[0], url} - - if !yield(item, nil) { - return - } - } + if err != nil { + log.Fatal(err) } -} - -func (m *ManifestsVar) Set(value string) error { - csv_path, err := runfiles.Rlocation(value) - if err != nil { log.Fatal(err) } - - m.path = csv_path - csv_byt, err := os.ReadFile(csv_path) - if err != nil { log.Fatal(err) } - - m.reader = csv.NewReader(strings.NewReader(string(csv_byt))) + *r = RunfileVar(runfile) return nil } -func upload(curl RunfileVar, src string, dest string, retry uint64, retry_delay uint64) error { +func upload(curl string, src string, dest string, retry uint64, retry_delay uint64) error { args := []string{ "--netrc", "--location", @@ -90,21 +36,25 @@ func upload(curl RunfileVar, src string, dest string, retry uint64, retry_delay "--retry", strconv.FormatUint(retry, 10), "--retry-delay", strconv.FormatUint(retry_delay, 10), "--upload-file", src, - dest, + "--url", dest, } - cmd := exec.Command(curl.String(), args...) - + cmd := exec.Command(curl, args...) + cmd.Stdout = os.Stdout err := cmd.Run() - if err != nil { return err } - + if err != nil { + return err + } + return nil } func setArgs(args_file string) error { args_byt, err := os.ReadFile(args_file) - if err != nil { return err } + if err != nil { + return err + } args := strings.Split(string(args_byt), "\n") // Remove ending newline @@ -112,8 +62,11 @@ func setArgs(args_file string) error { for i, _ := range args { if i%2 != 0 { - err = flag.Set(args[i-1], args[i]) - if err != nil { return err } + unquoted, err := shlex.Split(args[i]) + err = flag.Set(args[i-1], unquoted[0]) + if err != nil { + return err + } } } @@ -122,30 +75,33 @@ func setArgs(args_file string) error { func main() { var curl RunfileVar - var manifests ManifestsVar var retry uint64 var retry_delay uint64 - + var src RunfileVar + var url string + flag.Var(&curl, "curl", "The path to the curl bin") - flag.Var(&manifests, "csv", "The path to the csv") flag.Uint64Var(&retry, "retry", 0, "The number of times to retry the request") flag.Uint64Var(&retry_delay, "retry-delay", 1, "The number of seconds to wait before retrying the request") - + + flag.Var(&src, "src", "Path to the source file") + flag.StringVar(&url, "url", "", "The URL to upload to") + args_file, err := runfiles.Rlocation("upload.args") - if err != nil { log.Fatal(err) } + if err != nil { + log.Fatal(err) + } err = setArgs(args_file) - if err != nil { log.Fatal(err) } + if err != nil { + log.Fatal(err) + } flag.Parse() - - for upload_info, err := range manifests.Iterate() { - if err != nil { log.Fatal(err) } - - src, err := runfiles.Rlocation(upload_info[0]) - if err != nil { log.Fatal(err) } - - err = upload(curl, src, upload_info[1], retry, retry_delay) - if err != nil { log.Fatal(err) } + + err = upload(curl.String(), src.String(), url, retry, retry_delay) + if err != nil { + log.Fatal(err) } + } diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index 5c6a6de..cd70dcf 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -29,6 +29,9 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.31.0/MODULE.bazel": "0319690246f72d0b5d596724a0ea0da2fd823905643a042c95bc2c420438ddae", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", @@ -58,6 +61,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.6/MODULE.bazel": "5af9b4750785ee7603e4616a7463e87cb3ef016ec86da8cb6d592a33c735b6c0", "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.6/source.json": "58c2d1082dce218af90bcb70eddcb3b6b011dbe9168134614022f825f722a8db", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", diff --git a/e2e/upload/file/BUILD.bazel b/e2e/upload/file/BUILD.bazel index 0cdb043..655ce77 100644 --- a/e2e/upload/file/BUILD.bazel +++ b/e2e/upload/file/BUILD.bazel @@ -1,12 +1,31 @@ load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + +genrule( + name = "file_to_upload", + outs = [ + "input.txt", + ], + cmd = "echo TEST > $@", +) + +genrule( + name = "file_to_upload_no_extension", + outs = [ + "input", + ], + cmd = "echo TEST > $@", +) + + + curl_upload_file( name = "upload", testonly = True, - src = ":fixture.txt", - dst = "directory/fixture.txt", - url = "https://test.case", + src = ":file_to_upload", + url = "https://test.case/directory/fixture.txt", ) genrule( @@ -26,13 +45,11 @@ diff_file_test( b = ":execute", ) - curl_upload_file( name = "upload_no_extension", testonly = True, - src = ":fixture.txt", - dst = "directory/fixture", - url = "https://test.case", + src = ":file_to_upload_no_extension", + url = "https://test.case/directory/fixture", ) genrule( @@ -51,3 +68,61 @@ diff_file_test( a = ":fixture_no_extension.txt", b = ":execute_no_extension", ) + +string_flag( + name = "injection", + build_setting_default = "injection/", + visibility = ["//visibility:public"], + make_variable = "INJECT" +) + +curl_upload_file( + name = "upload_injection", + testonly = True, + src = ":file_to_upload", + url = "https://test.case/directory/$(INJECT)fixture.txt", + toolchains = [":injection"] +) + +genrule( + name = "execute_injection", + testonly = True, + outs = [ + "upload_injection.out", + ], + cmd = "./$(location :upload_injection) > $@", + tools = [":upload_injection"], +) + +diff_file_test( + name = "test_injection", + size = "small", + a = ":fixture_injection.txt", + b = ":execute_injection", +) + + +curl_upload_file( + name = "upload_original_filename", + testonly = True, + src = ":file_to_upload", + url = "https://test.case/directory/", +) + + +genrule( + name = "execute_original_filename", + testonly = True, + outs = [ + "upload_original_filename.out", + ], + cmd = "./$(location :upload_original_filename) > $@", + tools = [":upload_original_filename"], +) + +diff_file_test( + name = "test_original_filename", + size = "small", + a = ":fixture_original_filename.txt", + b = ":execute_original_filename", +) diff --git a/e2e/upload/file/fixture.txt b/e2e/upload/file/fixture.txt index 52ca57b..db6b94c 100644 --- a/e2e/upload/file/fixture.txt +++ b/e2e/upload/file/fixture.txt @@ -6,5 +6,6 @@ --retry-delay 1 --upload-file -fixture.txt +input.txt +--url https://test.case/directory/fixture.txt diff --git a/e2e/upload/file/fixture_injection.txt b/e2e/upload/file/fixture_injection.txt new file mode 100644 index 0000000..f4f08ab --- /dev/null +++ b/e2e/upload/file/fixture_injection.txt @@ -0,0 +1,11 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +input.txt +--url +https://test.case/directory/injection/fixture.txt diff --git a/e2e/upload/file/fixture_no_extension.txt b/e2e/upload/file/fixture_no_extension.txt index 806c14a..c503ff4 100644 --- a/e2e/upload/file/fixture_no_extension.txt +++ b/e2e/upload/file/fixture_no_extension.txt @@ -6,5 +6,6 @@ --retry-delay 1 --upload-file -fixture.txt +input +--url https://test.case/directory/fixture diff --git a/e2e/upload/file/fixture_original_filename.txt b/e2e/upload/file/fixture_original_filename.txt new file mode 100644 index 0000000..68cf35f --- /dev/null +++ b/e2e/upload/file/fixture_original_filename.txt @@ -0,0 +1,11 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +input.txt +--url +https://test.case/directory/ -- GitLab From f5f9397976eab835b98aabba8e7eb42ea9717abc Mon Sep 17 00:00:00 2001 From: Jonathan Watson Date: Mon, 27 Jan 2025 11:44:20 +0000 Subject: [PATCH 4/4] refactor: simplify repeated test pattern --- e2e/upload/file/BUILD.bazel | 106 +++++------------- .../file/{fixture.txt => expected_basic.txt} | 0 ...e_injection.txt => expected_injection.txt} | 0 ...xtension.txt => expected_no_extension.txt} | 0 ...ame.txt => expected_original_filename.txt} | 0 5 files changed, 28 insertions(+), 78 deletions(-) rename e2e/upload/file/{fixture.txt => expected_basic.txt} (100%) rename e2e/upload/file/{fixture_injection.txt => expected_injection.txt} (100%) rename e2e/upload/file/{fixture_no_extension.txt => expected_no_extension.txt} (100%) rename e2e/upload/file/{fixture_original_filename.txt => expected_original_filename.txt} (100%) diff --git a/e2e/upload/file/BUILD.bazel b/e2e/upload/file/BUILD.bazel index 655ce77..d3e917f 100644 --- a/e2e/upload/file/BUILD.bazel +++ b/e2e/upload/file/BUILD.bazel @@ -1,11 +1,10 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") - genrule( name = "file_to_upload", - outs = [ + outs = [ "input.txt", ], cmd = "echo TEST > $@", @@ -13,38 +12,19 @@ genrule( genrule( name = "file_to_upload_no_extension", - outs = [ + outs = [ "input", ], cmd = "echo TEST > $@", ) - - curl_upload_file( - name = "upload", + name = "upload_basic", testonly = True, src = ":file_to_upload", url = "https://test.case/directory/fixture.txt", ) -genrule( - name = "execute", - testonly = True, - outs = [ - "upload.out", - ], - cmd = "./$(location :upload) > $@", - tools = [":upload"], -) - -diff_file_test( - name = "test", - size = "small", - a = ":fixture.txt", - b = ":execute", -) - curl_upload_file( name = "upload_no_extension", testonly = True, @@ -52,56 +32,23 @@ curl_upload_file( url = "https://test.case/directory/fixture", ) -genrule( - name = "execute_no_extension", - testonly = True, - outs = [ - "upload_no_extension.out", - ], - cmd = "./$(location :upload_no_extension) > $@", - tools = [":upload_no_extension"], -) - -diff_file_test( - name = "test_no_extension", - size = "small", - a = ":fixture_no_extension.txt", - b = ":execute_no_extension", -) - string_flag( name = "injection", build_setting_default = "injection/", + make_variable = "INJECT", visibility = ["//visibility:public"], - make_variable = "INJECT" ) curl_upload_file( name = "upload_injection", testonly = True, src = ":file_to_upload", + toolchains = [":injection"], url = "https://test.case/directory/$(INJECT)fixture.txt", - toolchains = [":injection"] -) - -genrule( - name = "execute_injection", - testonly = True, - outs = [ - "upload_injection.out", - ], - cmd = "./$(location :upload_injection) > $@", - tools = [":upload_injection"], -) - -diff_file_test( - name = "test_injection", - size = "small", - a = ":fixture_injection.txt", - b = ":execute_injection", ) - +# We expect the url passed to curl to just be the url with a trailing +# slash, without a filename. That way curl will use the original file. curl_upload_file( name = "upload_original_filename", testonly = True, @@ -109,20 +56,23 @@ curl_upload_file( url = "https://test.case/directory/", ) - -genrule( - name = "execute_original_filename", - testonly = True, - outs = [ - "upload_original_filename.out", - ], - cmd = "./$(location :upload_original_filename) > $@", - tools = [":upload_original_filename"], -) - -diff_file_test( - name = "test_original_filename", - size = "small", - a = ":fixture_original_filename.txt", - b = ":execute_original_filename", -) +[ + ( + genrule( + name = "execute_{}".format(test), + testonly = True, + outs = [ + "upload_{}.out".format(test), + ], + cmd = "./$(location :upload_{}) > $@".format(test), + tools = [":upload_{}".format(test)], + ), + diff_file_test( + name = "test_{}".format(test), + size = "small", + a = ":expected_{}.txt".format(test), + b = ":execute_{}".format(test), + ), + ) + for test in ["basic", "original_filename", "injection", "no_extension"] +] diff --git a/e2e/upload/file/fixture.txt b/e2e/upload/file/expected_basic.txt similarity index 100% rename from e2e/upload/file/fixture.txt rename to e2e/upload/file/expected_basic.txt diff --git a/e2e/upload/file/fixture_injection.txt b/e2e/upload/file/expected_injection.txt similarity index 100% rename from e2e/upload/file/fixture_injection.txt rename to e2e/upload/file/expected_injection.txt diff --git a/e2e/upload/file/fixture_no_extension.txt b/e2e/upload/file/expected_no_extension.txt similarity index 100% rename from e2e/upload/file/fixture_no_extension.txt rename to e2e/upload/file/expected_no_extension.txt diff --git a/e2e/upload/file/fixture_original_filename.txt b/e2e/upload/file/expected_original_filename.txt similarity index 100% rename from e2e/upload/file/fixture_original_filename.txt rename to e2e/upload/file/expected_original_filename.txt -- GitLab