From d69266a8eb7e14d4a64a2184f8857e580ab88e4f Mon Sep 17 00:00:00 2001 From: Luke Hackwell Date: Tue, 14 Jan 2025 15:23:15 +0000 Subject: [PATCH 1/3] refactor: Remove POSIX/NT launcher scripts --- MODULE.bazel | 3 + curl/template/BUILD.bazel | 6 +- curl/template/template.go | 51 ++--------- curl/upload/BUILD.bazel | 26 +++--- curl/upload/file/rule.bzl | 53 ++++++------ curl/upload/manifests/rule.bzl | 49 +++++------ curl/upload/nt.tmpl.bat | 4 - curl/upload/posix.tmpl.sh | 63 -------------- curl/upload/upload.go | 151 +++++++++++++++++++++++++++++++++ 9 files changed, 224 insertions(+), 182 deletions(-) delete mode 100644 curl/upload/nt.tmpl.bat delete mode 100644 curl/upload/posix.tmpl.sh create mode 100644 curl/upload/upload.go diff --git a/MODULE.bazel b/MODULE.bazel index 8bf7bb2..bfb3e2e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,6 +10,9 @@ bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.18") bazel_dep(name = "ape", version = "1.0.0-beta.17") bazel_dep(name = "rules_go", version = "0.48.1") +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.23.4") + export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export") use_repo(export, "ape-curl") export.symlink( diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel index d22934d..b27161b 100644 --- a/curl/template/BUILD.bazel +++ b/curl/template/BUILD.bazel @@ -1,8 +1,8 @@ -load("@rules_go//go:def.bzl", "go_binary") +load("@rules_go//go:def.bzl", "go_library") -go_binary( +go_library( name = "template", srcs = ["template.go"], - pure = "on", + importpath = "gitlab.arm.com/bazel/rules_curl/curl/template", visibility = ["//curl/upload:__subpackages__"], ) diff --git a/curl/template/template.go b/curl/template/template.go index 1a0d740..5cb4062 100644 --- a/curl/template/template.go +++ b/curl/template/template.go @@ -1,9 +1,7 @@ -package main +package template import ( "bytes" - "flag" - "fmt" "html/template" "log" "net/url" @@ -12,11 +10,6 @@ import ( "strings" ) -var manifest ManifestInfo = ManifestInfo{ - URL: URLInfo{}, - File: FileInfo{}, -} - type StringInfo string func (s StringInfo) String() string { @@ -26,12 +19,7 @@ func (s StringInfo) String() string { func (s *StringInfo) Set(value string) error { buf := new(bytes.Buffer) - tmpl, err := template.New("templ").Parse(value) - if err != nil { - log.Fatal(err) - } - - err = tmpl.Execute(buf, manifest) + _, err := template.New("templ").Parse(value) if err != nil { log.Fatal(err) } @@ -164,35 +152,8 @@ type ManifestInfo struct { File FileInfo } -func main() { - flag.Var(&manifest.File, "file", "The file path to use for templating") - flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file") - flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file") - flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file") - flag.Var(&manifest.URL, "url", "URL to use for templating") - flag.Var(&manifest.URL.Host, "host", "The domain name of the URL") - flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL") - flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error { - return manifest.URL.Pathname.Set(s) - }) - flag.Func("origin", "The origin of the represented URL.", func(s string) error { - index := strings.Index(s, "//") - - if index == -1 { - manifest.URL.Protocol = StringInfo(s) - return nil - } - - manifest.URL.Host = StringInfo(s[index+2:]) - manifest.URL.Protocol = StringInfo(s[:index]) - - return nil - }) - t := flag.String("template", "{{.URL.Href}}/{{.File.Path}}", "The Go template to render") - - flag.Parse() - - tmpl, err := template.New("url").Parse(*t) +func Tmpl(manifest ManifestInfo, template_s string) string { + tmpl, err := template.New("url").Parse(template_s) if err != nil { log.Fatal(err) } @@ -208,6 +169,6 @@ func main() { if err != nil { log.Fatal(err) } - - fmt.Printf("%s", buf.String()) + + return buf.String() } diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 04169ed..4e648fb 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -1,22 +1,18 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) +load("@rules_go//go:def.bzl", "go_binary") -alias( - name = "script", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_manifests`", - ), - visibility = ["//curl/upload:__subpackages__"], -) sh_binary( name = "csv", srcs = ["csv.sh"], visibility = ["//curl/upload:__subpackages__"], ) + +go_binary( + name = "upload", + srcs = ["upload.go"], + visibility = ["//curl/upload:__subpackages__"], + deps = [ + "//curl/template", + "@rules_go//go/runfiles:go_default_library", + ], +) diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index ddad0e1..e572ffa 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -36,16 +36,11 @@ ATTRS = { doc = "The seconds to wait before attempting a upload retry.", default = 1, ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", + "_upload": attr.label( + default = "//curl/upload", allow_single_file = True, executable = True, + cfg = "exec" ), "_csv": attr.label( doc = "CSV tool", @@ -73,30 +68,32 @@ def implementation(ctx): mnemonic = "PrepareUploadCSV", ) - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - substitutions.add("{{directory}}", str(ctx.file.src.short_path)) - - ctx.actions.expand_template( - template = ctx.file._script, + arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) + args = ctx.actions.args() + args.add("curl", curl.executable) + args.add("csv", csv.short_path) + args.add("retry", str(ctx.attr.retry)) + args.add("retry-delay", str(ctx.attr.retry_delay)) + ctx.actions.write(output = arguments, content = args) + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( output = executable, - computed_substitutions = substitutions, + target_file = ctx.executable._upload, is_executable = True, ) - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, ctx.file.src, csv]) - runfiles = runfiles.merge(ctx.attr.src.default_runfiles) - runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) - runfiles = runfiles.merge(ctx.attr._csv.default_runfiles) - + files = depset([executable, arguments]) + + symlinks = { + "upload.args": arguments, + curl.executable.short_path: curl.executable, + csv.short_path: csv, + ctx.file.src.short_path: ctx.file.src, + } + + runfiles = ctx.runfiles([], root_symlinks = symlinks) + return DefaultInfo( executable = executable, files = files, diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 713ebd9..05cd7b5 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -40,16 +40,11 @@ ATTRS = { doc = "The seconds to wait before attempting an upload retry.", default = 1, ), - "_script": attr.label( - doc = "The template that is expanded into the upload binary.", - default = "//curl/upload:script", - allow_single_file = True, - ), - "_template": attr.label( - default = "//curl/template:template", - cfg = "exec", + "_upload": attr.label( + default = "//curl/upload", allow_single_file = True, executable = True, + cfg = "exec", ), "_csv": attr.label( doc = "CSV tool", @@ -84,26 +79,32 @@ def implementation(ctx): mnemonic = "PrepareUploadCSV", ) - executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - substitutions = ctx.actions.template_dict() - substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{retry}}", str(ctx.attr.retry)) - substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{csv}}", str(csv.short_path)) - substitutions.add("{{template}}", str(ctx.executable._template.short_path)) - - ctx.actions.expand_template( - template = ctx.file._script, + arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) + args = ctx.actions.args() + args.add("curl", curl.executable) + args.add("csv", csv.short_path) + args.add("retry", str(ctx.attr.retry)) + args.add("retry-delay", str(ctx.attr.retry_delay)) + ctx.actions.write(output = arguments, content = args) + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( output = executable, - computed_substitutions = substitutions, + target_file = ctx.executable._upload, is_executable = True, ) - files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, csv, ctx.executable._template] + [m.file for m in manifests.to_list()]) - runfiles = runfiles.merge(curl.default.default_runfiles) - runfiles = runfiles.merge(ctx.attr._template.default_runfiles) + files = depset([executable, arguments]) + + symlinks = { + "upload.args": arguments, + curl.executable.short_path: curl.executable, + csv.short_path: csv, + } + for manifest in manifests.to_list(): + symlinks.update({manifest.file.short_path: manifest.file}) + + runfiles = ctx.runfiles([], root_symlinks = symlinks) return DefaultInfo( executable = executable, diff --git a/curl/upload/nt.tmpl.bat b/curl/upload/nt.tmpl.bat deleted file mode 100644 index d522214..0000000 --- a/curl/upload/nt.tmpl.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -# TODO: implement Windows Batch for `curl_upload` -exit /b 121 diff --git a/curl/upload/posix.tmpl.sh b/curl/upload/posix.tmpl.sh deleted file mode 100644 index 05687f3..0000000 --- a/curl/upload/posix.tmpl.sh +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env sh - -# Strict shell -set -o errexit -o nounset - -# Runfiles location -rlocation() ( - readonly FILEPATH="${1}" - - if test -e "${FILEPATH}"; then - printf '%s' "${FILEPATH}" - return - fi - - readonly RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" - - if test -e "${RUNFILES_DIR}/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/${FILEPATH#../}" - return - fi - - if test -e "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then - printf '%s' "${RUNFILES_DIR}/"*"/${FILEPATH#../}" - return - fi - - printf >&2 "No runfile found: %s\n" "${FILEPATH}" - exit 1 -) - -# Bazel substitutions -CURL="$(rlocation "{{curl}}")" -CSV="$(rlocation "{{csv}}")" -TEMPLATE="$(rlocation "{{template}}")" -RETRY="{{retry}}" -RETRY_DELAY="{{retry_delay}}" -readonly CURL CSV TEMPLATE RETRY RETRY_DELAY - -# Uploads a file -upload() { - printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" - - # Do the upload - "${CURL}" \ - --netrc \ - --location \ - --progress-bar \ - --retry "${RETRY}" \ - --retry-delay "${RETRY_DELAY}" \ - --upload-file "$(rlocation "${1}")" \ - "${2}" -} - -while IFS=, read -r SRC DST TMPL URL; do - FULL_DST=$( - "${TEMPLATE}" \ - --url "${URL}" \ - --template "${TMPL}" \ - --file "${DST}" \ - ${@} - ) - upload "${SRC}" "${FULL_DST}" -done <"${CSV}" diff --git a/curl/upload/upload.go b/curl/upload/upload.go new file mode 100644 index 0000000..8985d15 --- /dev/null +++ b/curl/upload/upload.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/csv" + "flag" + "io" + "iter" + "os" + "log" + "os/exec" + "strconv" + "strings" + "gitlab.arm.com/bazel/rules_curl/curl/template" + "github.com/bazelbuild/rules_go/go/runfiles" +) + +type RunfileVar string + +func (r RunfileVar) String() string { + return string(r) +} + +func (r *RunfileVar) Set(value string) error { + runfile, err := runfiles.Rlocation(value) + if err != nil { log.Fatal(err) } + + *r = RunfileVar(runfile) + return nil +} + +type ManifestsVar struct { + reader *csv.Reader + path string +} + +func (m ManifestsVar) String() string { + return m.path +} + +func (m ManifestsVar) Iterate() iter.Seq2[[2]string, error] { + return func(yield func([2]string, error) bool) { + var empty [2]string + for { + record, err := m.reader.Read() + if err == io.EOF { + return + } + if err != nil { + if !yield(empty, err) { + return + } + } + + var manifest template.ManifestInfo = template.ManifestInfo { + URL: template.URLInfo{}, + File: template.FileInfo{}, + } + + manifest.URL.Set(record[3]) + manifest.File.Set(record[1]) + url := template.Tmpl(manifest, record[2]) + + item := [2]string{record[0], url} + + if !yield(item, nil) { + return + } + } + } +} + +func (m *ManifestsVar) Set(value string) error { + csv_path, err := runfiles.Rlocation(value) + if err != nil { log.Fatal(err) } + + m.path = csv_path + + csv_byt, err := os.ReadFile(csv_path) + if err != nil { log.Fatal(err) } + + m.reader = csv.NewReader(strings.NewReader(string(csv_byt))) + return nil +} + +func upload(curl RunfileVar, src string, dest string, retry uint64, retry_delay uint64) error { + args := []string{ + "--netrc", + "--location", + "--progress-bar", + "--retry", strconv.FormatUint(retry, 10), + "--retry-delay", strconv.FormatUint(retry_delay, 10), + "--upload-file", src, + dest, + } + cmd := exec.Command(curl.String(), args...) + + cmd.Stdout = os.Stdout + + err := cmd.Run() + if err != nil { return err } + + return nil +} + +func setArgs(args_file string) error { + args_byt, err := os.ReadFile(args_file) + if err != nil { return err } + + args := strings.Split(string(args_byt), "\n") + // Remove ending newline + args = args[:len(args)-1] + + for i, _ := range args { + if i%2 != 0 { + err = flag.Set(args[i-1], args[i]) + if err != nil { return err } + } + } + + return nil +} + +func main() { + var curl RunfileVar + var manifests ManifestsVar + var retry uint64 + var retry_delay uint64 + + flag.Var(&curl, "curl", "The path to the curl bin") + flag.Var(&manifests, "csv", "The path to the csv") + flag.Uint64Var(&retry, "retry", 0, "The number of times to retry the request") + flag.Uint64Var(&retry_delay, "retry-delay", 1, "The number of seconds to wait before retrying the request") + + args_file, err := runfiles.Rlocation("upload.args") + if err != nil { log.Fatal(err) } + + err = setArgs(args_file) + if err != nil { log.Fatal(err) } + + flag.Parse() + + for upload_info, err := range manifests.Iterate() { + if err != nil { log.Fatal(err) } + + src, err := runfiles.Rlocation(upload_info[0]) + if err != nil { log.Fatal(err) } + + err = upload(curl, src, upload_info[1], retry, retry_delay) + if err != nil { log.Fatal(err) } + } +} -- GitLab From 4dd2dfc71cb325fb92ab35b2fc07f783d4ebc487 Mon Sep 17 00:00:00 2001 From: Luke Hackwell Date: Mon, 20 Jan 2025 09:28:45 +0000 Subject: [PATCH 2/3] fix: replace symlinks with runfiles --- curl/upload/file/rule.bzl | 28 ++++++++++++++++------------ curl/upload/manifests/rule.bzl | 27 +++++++++++++++------------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index e572ffa..030adf1 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -50,6 +50,13 @@ ATTRS = { ), } + +def _runfile(label, file): + path = file.short_path + if path.startswith("../"): + return path.removeprefix("../") + return "{}/{}".format(label.workspace_name or "_main", path) + def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] @@ -58,7 +65,9 @@ def implementation(ctx): args = ctx.actions.args() # SRC, DST, TEMPLATE, URL - args.add("{},{},{},{}".format(ctx.file.src.short_path, ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) + args.add("{},{},{},{}".format( + _runfile(ctx.file.src.owner, ctx.file.src), + ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href)) ctx.actions.run( outputs = [csv], @@ -67,11 +76,11 @@ def implementation(ctx): executable = ctx.executable._csv, mnemonic = "PrepareUploadCSV", ) - arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) args = ctx.actions.args() - args.add("curl", curl.executable) - args.add("csv", csv.short_path) + + args.add("curl", _runfile(curl.executable.owner, curl.executable)) + args.add("csv", _runfile(csv.owner, csv)) args.add("retry", str(ctx.attr.retry)) args.add("retry-delay", str(ctx.attr.retry_delay)) ctx.actions.write(output = arguments, content = args) @@ -84,15 +93,10 @@ def implementation(ctx): ) files = depset([executable, arguments]) + root_symlinks = { "upload.args": arguments } - symlinks = { - "upload.args": arguments, - curl.executable.short_path: curl.executable, - csv.short_path: csv, - ctx.file.src.short_path: ctx.file.src, - } - - runfiles = ctx.runfiles([], root_symlinks = symlinks) + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv, ctx.file.src], root_symlinks = root_symlinks) + runfiles = runfiles.merge(curl.default.default_runfiles) return DefaultInfo( executable = executable, diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 05cd7b5..74b6bf1 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -54,6 +54,12 @@ ATTRS = { ), } +def _runfile(label, file): + path = file.short_path + if path.startswith("../"): + return path.removeprefix("../") + return "{}/{}".format(label.workspace_name or "_main", path) + def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] @@ -67,7 +73,8 @@ def implementation(ctx): def _to_string(m): # SRC, DST, TEMPLATE, URL - return "{},{},{},{}".format(m.file.short_path, m.file.short_path, m.url, href) + return "{},{},{},{}".format( + _runfile(m.file.owner, m.file), m.file.short_path, m.url, href) args = ctx.actions.args() args.add_all(manifests, map_each = _to_string, allow_closure = True) @@ -81,12 +88,12 @@ def implementation(ctx): arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name)) args = ctx.actions.args() - args.add("curl", curl.executable) - args.add("csv", csv.short_path) + args.add("curl", _runfile(curl.executable.owner, curl.executable)) + args.add("csv", _runfile(csv.owner, csv)) args.add("retry", str(ctx.attr.retry)) args.add("retry-delay", str(ctx.attr.retry_delay)) ctx.actions.write(output = arguments, content = args) - + executable = ctx.actions.declare_file(ctx.label.name) ctx.actions.symlink( output = executable, @@ -95,16 +102,12 @@ def implementation(ctx): ) files = depset([executable, arguments]) + root_symlinks = { "upload.args": arguments } - symlinks = { - "upload.args": arguments, - curl.executable.short_path: curl.executable, - csv.short_path: csv, - } + runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, csv], root_symlinks = root_symlinks) + runfiles = runfiles.merge(curl.default.default_runfiles) for manifest in manifests.to_list(): - symlinks.update({manifest.file.short_path: manifest.file}) - - runfiles = ctx.runfiles([], root_symlinks = symlinks) + runfiles = runfiles.merge(ctx.runfiles([manifest.file])) return DefaultInfo( executable = executable, -- GitLab From bc922f370961da57545c8ef5c26022241dbadaf9 Mon Sep 17 00:00:00 2001 From: Luke Hackwell Date: Mon, 20 Jan 2025 11:13:39 +0000 Subject: [PATCH 3/3] fix: CI failure --- curl/upload/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 4e648fb..9398349 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -15,4 +15,5 @@ go_binary( "//curl/template", "@rules_go//go/runfiles:go_default_library", ], + pure = "on" ) -- GitLab