From f805b9163fe7d46db8f94d98a000e2acfaf0c4eb Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 08:56:54 +0200 Subject: [PATCH 1/7] fix: non-executable runfiles Runfiles do not have to be executable. --- curl/upload/manifests/posix.tmpl.sh | 7 ++++--- e2e/mock/curl.sh | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/manifests/posix.tmpl.sh index d11c9ea..632ea48 100644 --- a/curl/upload/manifests/posix.tmpl.sh +++ b/curl/upload/manifests/posix.tmpl.sh @@ -7,23 +7,24 @@ set -o errexit -o nounset rlocation() ( readonly FILEPATH="${1}" - if test -x "${FILEPATH}"; then + if test -e "${FILEPATH}"; then printf '%s' "${FILEPATH}" return fi readonly RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" - if test -x "${RUNFILES_DIR}/${FILEPATH#../}"; then + if test -e "${RUNFILES_DIR}/${FILEPATH#../}"; then printf '%s' "${RUNFILES_DIR}/${FILEPATH#../}" return fi - if test -x "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then + if test -e "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then printf '%s' "${RUNFILES_DIR}/"*"/${FILEPATH#../}" return fi + printf >&2 "No runfile found: %s\n" "${FILEPATH}" exit 1 ) diff --git a/e2e/mock/curl.sh b/e2e/mock/curl.sh index 2eba66d..1357249 100755 --- a/e2e/mock/curl.sh +++ b/e2e/mock/curl.sh @@ -9,7 +9,7 @@ while test "${#}" -ne 0; do "--upload-file") printf '%s\n' "${1}" shift - if ! test -f "${1}"; then + if ! test -e "${1}"; then printf >&2 'Not found: %s\n' "${1}" exit 2 fi -- GitLab From cffc6734e2fbee2ecaa4567a699505cd734795ac Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 09:49:40 +0200 Subject: [PATCH 2/7] fix: template `pathname` Passing correct value is a user responsibility. `template` is not aware if provided data is correct or wrong. --- curl/template/template.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/curl/template/template.go b/curl/template/template.go index fc2a054..8d4f7fc 100644 --- a/curl/template/template.go +++ b/curl/template/template.go @@ -168,15 +168,7 @@ func main() { flag.Var(&manifest.URL.Host, "host", "The domain name of the URL") flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL") flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error { - buf := new(bytes.Buffer) - if !strings.HasPrefix(s, "/") { - buf.WriteString("/") - } - buf.WriteString(s) - - manifest.URL.Pathname.Set(buf.String()) - - return nil + return manifest.URL.Pathname.Set(s) }) flag.Func("origin", "The origin of the represented URL.", func(s string) error { index := strings.Index(s, "//") -- GitLab From 5defc18431dae2afd07d01853addae484f9ce3d0 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 10:57:04 +0200 Subject: [PATCH 3/7] refactor: move scripts level up This will allow both rules use the same set of scripts. --- curl/upload/BUILD.bazel | 22 ++++++++++++++++++++++ curl/upload/{manifests => }/csv.sh | 0 curl/upload/manifests/BUILD.bazel | 22 ---------------------- curl/upload/manifests/rule.bzl | 4 ++-- curl/upload/{manifests => }/nt.tmpl.bat | 0 curl/upload/{manifests => }/posix.tmpl.sh | 0 6 files changed, 24 insertions(+), 24 deletions(-) rename curl/upload/{manifests => }/csv.sh (100%) rename curl/upload/{manifests => }/nt.tmpl.bat (100%) rename curl/upload/{manifests => }/posix.tmpl.sh (100%) diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index e69de29..9f3bdb7 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -0,0 +1,22 @@ +exports_files([ + "posix.tmpl.sh", + "nt.tmpl.bat", +]) + +alias( + name = "script", + actual = select( + { + "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", + "//conditions:default": ":posix.tmpl.sh", + }, + no_match_error = "No script template available for `curl_upload_manifests`", + ), + visibility = ["//curl/upload/manifests:__pkg__"], +) + +sh_binary( + name = "csv", + srcs = ["csv.sh"], + visibility = ["//curl/upload/manifests:__pkg__"], +) diff --git a/curl/upload/manifests/csv.sh b/curl/upload/csv.sh similarity index 100% rename from curl/upload/manifests/csv.sh rename to curl/upload/csv.sh diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel index 9f3bdb7..e69de29 100644 --- a/curl/upload/manifests/BUILD.bazel +++ b/curl/upload/manifests/BUILD.bazel @@ -1,22 +0,0 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) - -alias( - name = "script", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_manifests`", - ), - visibility = ["//curl/upload/manifests:__pkg__"], -) - -sh_binary( - name = "csv", - srcs = ["csv.sh"], - visibility = ["//curl/upload/manifests:__pkg__"], -) diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 9a4ac1c..0235ef1 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -42,7 +42,7 @@ ATTRS = { ), "_script": attr.label( doc = "The template that is expanded into the upload binary.", - default = ":script", + default = "//curl/upload:script", allow_single_file = True, ), "_template": attr.label( @@ -53,7 +53,7 @@ ATTRS = { ), "_csv": attr.label( doc = "CSV tool", - default = ":csv", + default = "//curl/upload:csv", cfg = "exec", executable = True, ), diff --git a/curl/upload/manifests/nt.tmpl.bat b/curl/upload/nt.tmpl.bat similarity index 100% rename from curl/upload/manifests/nt.tmpl.bat rename to curl/upload/nt.tmpl.bat diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/posix.tmpl.sh similarity index 100% rename from curl/upload/manifests/posix.tmpl.sh rename to curl/upload/posix.tmpl.sh -- GitLab From 280e5b91eec50fec5a5d87f7aeb2f68873513a17 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 11:01:09 +0200 Subject: [PATCH 4/7] fix: labeled csv file name Created `upload.csv` file is unique. --- curl/upload/manifests/rule.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 0235ef1..3e6f056 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -62,7 +62,7 @@ ATTRS = { def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] - csv = ctx.actions.declare_file("upload.csv") + csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) manifests = depset( direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], -- GitLab From b9c048ba1783256434f1aabc9ee503dd2e851768 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 11:07:29 +0200 Subject: [PATCH 5/7] chore: add mnemonic Mnemonic for making `*.upload.cvs` file action. --- curl/upload/manifests/rule.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 3e6f056..1d1bdef 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -81,6 +81,7 @@ def implementation(ctx): outputs = [csv], arguments = [csv.path, args], executable = ctx.executable._csv, + mnemonic = "PrepareUploadCSV", ) executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) -- GitLab From 939299b0fa2249f626a55426e0ebeddcf554f455 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 11:09:24 +0200 Subject: [PATCH 6/7] fix: no empty lines Do not put empty lines to CSV file. --- curl/upload/csv.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/curl/upload/csv.sh b/curl/upload/csv.sh index 1543dd8..39301d0 100755 --- a/curl/upload/csv.sh +++ b/curl/upload/csv.sh @@ -7,5 +7,7 @@ shift readonly OUT for ARG in "${@}"; do - printf >>"${OUT}" '%s\n' "${ARG}" + if test -n "${ARG}"; then + printf >>"${OUT}" '%s\n' "${ARG}" + fi done -- GitLab From c3d93d1de282ae1d3ba80e448de84bd638737ef0 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 5 Jul 2024 11:23:11 +0200 Subject: [PATCH 7/7] refactor: unify rules BREAKING CHANGE: different set of `curl_upload_file` run arguments `curl_upload_file` and `curl_upload_manifests` are unified of uploading files. --- curl/upload/BUILD.bazel | 4 +- curl/upload/file/BUILD.bazel | 16 -------- curl/upload/file/nt.tmpl.bat | 4 -- curl/upload/file/posix.tmpl.sh | 75 ---------------------------------- curl/upload/file/rule.bzl | 45 ++++++++++++++++---- 5 files changed, 40 insertions(+), 104 deletions(-) delete mode 100644 curl/upload/file/nt.tmpl.bat delete mode 100644 curl/upload/file/posix.tmpl.sh diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel index 9f3bdb7..04169ed 100644 --- a/curl/upload/BUILD.bazel +++ b/curl/upload/BUILD.bazel @@ -12,11 +12,11 @@ alias( }, no_match_error = "No script template available for `curl_upload_manifests`", ), - visibility = ["//curl/upload/manifests:__pkg__"], + visibility = ["//curl/upload:__subpackages__"], ) sh_binary( name = "csv", srcs = ["csv.sh"], - visibility = ["//curl/upload/manifests:__pkg__"], + visibility = ["//curl/upload:__subpackages__"], ) diff --git a/curl/upload/file/BUILD.bazel b/curl/upload/file/BUILD.bazel index 284cdfa..e69de29 100644 --- a/curl/upload/file/BUILD.bazel +++ b/curl/upload/file/BUILD.bazel @@ -1,16 +0,0 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) - -alias( - name = "template", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_file`", - ), - visibility = ["//visibility:public"], -) diff --git a/curl/upload/file/nt.tmpl.bat b/curl/upload/file/nt.tmpl.bat deleted file mode 100644 index d522214..0000000 --- a/curl/upload/file/nt.tmpl.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -# TODO: implement Windows Batch for `curl_upload` -exit /b 121 diff --git a/curl/upload/file/posix.tmpl.sh b/curl/upload/file/posix.tmpl.sh deleted file mode 100644 index 59ee342..0000000 --- a/curl/upload/file/posix.tmpl.sh +++ /dev/null @@ -1,75 +0,0 @@ -#! /usr/bin/env sh - -# Strict shell -set -o errexit -o nounset - -# Bazel substitutions -CURL="{{curl}}" -SRC="{{src}}" -DST="{{dst}}" -URL="{{url}}" -RETRY="{{retry}}" -RETRY_DELAY="{{retry_delay}}" -readonly CURL SRC DST URL RETRY RETRY_DELAY - -# Runfiles -RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" -REPO_MAPPING="${RUNFILES_DIR}/_repo_mapping" -readonly RUNFILES_DIR REPO_MAPPING - -# Repository Mapping -if test -f "${REPO_MAPPING}"; then - while IFS=, read -r CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL; do - if test "${TARGET_LOCAL}" != "__main__"; then - printf >&2 'Error: Unexpected repository mapping: %s,%s,%s\n' "${CURRENT_CANONICAL}" "${TARGET_LOCAL}" "${TARGET_CANONICAL}" - exit 2 - fi - RUNFILES="${RUNFILES_DIR}/${TARGET_CANONICAL}" - break - done <"${REPO_MAPPING}" - unset CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL -else - RUNFILES="${0%/*}" -fi - -# Parse arguments -ENDPOINT="${URL}" -DIRECTORY="/" -DESTINATION="${DST}" -while test 0 -ne "${#}"; do - case "${1}" in - "--url") - shift - ENDPOINT="${1?Must provide an argument for --url}" - ;; - "--dir" | "--directory") - shift - DIRECTORY="${DIRECTORY}${1?Must provide an argument for --directory}/" - ;; - "--dst" | "--destination") - shift - DESTINATION="${1?Must provide an argument for --destination}" - ;; - *) - printf >&2 'Error: unknown argument: %s\n' "${1}" - exit 2 - ;; - esac - shift -done -readonly ENDPOINT DIRECTORY DESTINATION - -COMPOSED="${ENDPOINT}${DIRECTORY}${DESTINATION}" -readonly COMPOSED - -printf >&2 "Uploading: %s to %s\n" "${SRC}" "${COMPOSED}" - -# Do the upload -"${RUNFILES}/${CURL}" \ - --netrc \ - --location \ - --progress-bar \ - --retry "${RETRY}" \ - --retry-delay "${RETRY_DELAY}" \ - --upload-file "${RUNFILES}/${SRC}" \ - "${COMPOSED}" diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index 01e149c..58f9ff9 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -1,3 +1,5 @@ +load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info") + visibility("//curl/...") DOC = """Upload a file to a URL endpoint with cURL. @@ -34,37 +36,66 @@ ATTRS = { doc = "The seconds to wait before attempting a upload retry.", default = 1, ), - "template": attr.label( + "_script": attr.label( doc = "The template that is expanded into the upload binary.", - default = ":template", + default = "//curl/upload:script", + allow_single_file = True, + ), + "_template": attr.label( + default = "//curl/template:template", + cfg = "exec", allow_single_file = True, + executable = True, ), + "_csv": attr.label( + doc = "CSV tool", + default = "//curl/upload:csv", + cfg = "exec", + executable = True, + ), } def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] + csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name)) + href = ctx.attr.url.rstrip("/").replace(",", "%2C") + dst = ctx.attr.dst + + args = ctx.actions.args() + args.add("{},{},{}".format(ctx.file.src.short_path, "{{{{.URL.Href}}}}/{}".format(dst), href)) + + ctx.actions.run( + outputs = [csv], + inputs = [ctx.file.src], + arguments = [csv.path, args], + executable = ctx.executable._csv, + mnemonic = "PrepareUploadCSV", + ) + executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) substitutions = ctx.actions.template_dict() substitutions.add("{{curl}}", str(curl.executable.short_path)) - substitutions.add("{{url}}", ctx.attr.url.rstrip("/")) substitutions.add("{{retry}}", str(ctx.attr.retry)) substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{src}}", str(ctx.file.src.short_path)) - substitutions.add("{{dst}}", str(ctx.attr.dst)) + substitutions.add("{{csv}}", str(csv.short_path)) + substitutions.add("{{template}}", str(ctx.executable._template.short_path)) + substitutions.add("{{directory}}", str(ctx.file.src.short_path)) ctx.actions.expand_template( - template = ctx.file.template, + template = ctx.file._script, output = executable, computed_substitutions = substitutions, is_executable = True, ) files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, ctx.file.src]) + runfiles = ctx.runfiles([curl.executable, ctx.file.src, csv]) runfiles = runfiles.merge(ctx.attr.src.default_runfiles) runfiles = runfiles.merge(curl.default.default_runfiles) + runfiles = runfiles.merge(ctx.attr._template.default_runfiles) + runfiles = runfiles.merge(ctx.attr._csv.default_runfiles) return DefaultInfo( executable = executable, -- GitLab