From b40c721ffb647cb35b10a9b794c2ecae7483e6d5 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Tue, 11 Jun 2024 09:39:14 +0200 Subject: [PATCH 1/8] feat: rule to upload bunch of files Provide new rule who accepts `UploadFileInfo` provider. This way all the files from the provider will be uploded. --- curl/upload/BUILD.bazel | 16 +++++ curl/upload/file/BUILD.bazel | 16 ----- curl/upload/file/rule.bzl | 3 +- curl/upload/files/BUILD.bazel | 0 curl/upload/files/defs.bzl | 5 ++ curl/upload/files/rule.bzl | 96 ++++++++++++++++++++++++++++ curl/upload/{file => }/nt.tmpl.bat | 0 curl/upload/{file => }/posix.tmpl.sh | 34 ++++++---- curl/upload/providers.bzl | 8 +++ 9 files changed, 150 insertions(+), 28 deletions(-) create mode 100644 curl/upload/BUILD.bazel create mode 100644 curl/upload/files/BUILD.bazel create mode 100644 curl/upload/files/defs.bzl create mode 100644 curl/upload/files/rule.bzl rename curl/upload/{file => }/nt.tmpl.bat (100%) rename curl/upload/{file => }/posix.tmpl.sh (71%) create mode 100644 curl/upload/providers.bzl diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel new file mode 100644 index 0000000..284cdfa --- /dev/null +++ b/curl/upload/BUILD.bazel @@ -0,0 +1,16 @@ +exports_files([ + "posix.tmpl.sh", + "nt.tmpl.bat", +]) + +alias( + name = "template", + actual = select( + { + "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", + "//conditions:default": ":posix.tmpl.sh", + }, + no_match_error = "No script template available for `curl_upload_file`", + ), + visibility = ["//visibility:public"], +) diff --git a/curl/upload/file/BUILD.bazel b/curl/upload/file/BUILD.bazel index 284cdfa..e69de29 100644 --- a/curl/upload/file/BUILD.bazel +++ b/curl/upload/file/BUILD.bazel @@ -1,16 +0,0 @@ -exports_files([ - "posix.tmpl.sh", - "nt.tmpl.bat", -]) - -alias( - name = "template", - actual = select( - { - "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", - "//conditions:default": ":posix.tmpl.sh", - }, - no_match_error = "No script template available for `curl_upload_file`", - ), - visibility = ["//visibility:public"], -) diff --git a/curl/upload/file/rule.bzl b/curl/upload/file/rule.bzl index 01e149c..c571123 100644 --- a/curl/upload/file/rule.bzl +++ b/curl/upload/file/rule.bzl @@ -36,7 +36,7 @@ ATTRS = { ), "template": attr.label( doc = "The template that is expanded into the upload binary.", - default = ":template", + default = "//curl/upload:template", allow_single_file = True, ), } @@ -53,6 +53,7 @@ def implementation(ctx): substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) substitutions.add("{{src}}", str(ctx.file.src.short_path)) substitutions.add("{{dst}}", str(ctx.attr.dst)) + substitutions.add("{{input_data}}", "") # must be empty ctx.actions.expand_template( template = ctx.file.template, diff --git a/curl/upload/files/BUILD.bazel b/curl/upload/files/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/curl/upload/files/defs.bzl b/curl/upload/files/defs.bzl new file mode 100644 index 0000000..6463a79 --- /dev/null +++ b/curl/upload/files/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _files = "files") + +visibility("public") + +curl_upload_files = _files diff --git a/curl/upload/files/rule.bzl b/curl/upload/files/rule.bzl new file mode 100644 index 0000000..a9572de --- /dev/null +++ b/curl/upload/files/rule.bzl @@ -0,0 +1,96 @@ +load("//curl/upload:providers.bzl", "UploadFilesInfo") + +visibility("//curl/...") + +DOC = """Upload bunch of files to a URL endpoint with cURL. + +The `srcs` must provide `UploadFilesInfo`. + +```py +file( + name = "upload_file", + srcs = [ + ":data", + ], + dst = "path/to/prefix" + url = "https://host.name.to.upload", +) +``` +""" + +ATTRS = { + "srcs": attr.label_list( + doc = "Files to be uploaded.", + mandatory = True, + providers = [UploadFilesInfo], + ), + "dst": attr.string( + doc = "The destination directory to upload.", + default = "", + ), + "url": attr.string( + doc = "URL endpoint for file upload.", + mandatory = True, + ), + "retry": attr.int( + doc = "The number of retry attempts.", + default = 3, + ), + "retry_delay": attr.int( + doc = "The seconds to wait before attempting a upload retry.", + default = 1, + ), + "template": attr.label( + doc = "The template that is expanded into the upload binary.", + default = "//curl/upload:template", + allow_single_file = True, + ), +} + +def implementation(ctx): + curl = ctx.toolchains["//curl/toolchain/curl:type"] + + content = "" + upload_data = ctx.actions.declare_file("upload_data.txt") + upload = depset(transitive = [src[UploadFilesInfo].files for src in ctx.attr.srcs]) + + for f in upload.to_list(): + content += "{}\n".format(f.short_path) + + ctx.actions.write(upload_data, content, is_executable=False) + + executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) + + substitutions = ctx.actions.template_dict() + substitutions.add("{{curl}}", str(curl.executable.short_path)) + substitutions.add("{{url}}", ctx.attr.url.rstrip("/")) + substitutions.add("{{retry}}", str(ctx.attr.retry)) + substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) + substitutions.add("{{src}}", "") + substitutions.add("{{dst}}", str(ctx.attr.dst)) + substitutions.add("{{upload_data}}", str(upload_data.short_path)) + + ctx.actions.expand_template( + template = ctx.file.template, + output = executable, + computed_substitutions = substitutions, + is_executable = True, + ) + + files = depset([executable]) + runfiles = ctx.runfiles([curl.executable, upload_data] + ctx.files.srcs + upload.to_list()) + runfiles = runfiles.merge(curl.default.default_runfiles) + + return DefaultInfo( + executable = executable, + files = files, + runfiles = runfiles, + ) + +files = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, + toolchains = ["//curl/toolchain/curl:type"], + executable = True, +) diff --git a/curl/upload/file/nt.tmpl.bat b/curl/upload/nt.tmpl.bat similarity index 100% rename from curl/upload/file/nt.tmpl.bat rename to curl/upload/nt.tmpl.bat diff --git a/curl/upload/file/posix.tmpl.sh b/curl/upload/posix.tmpl.sh similarity index 71% rename from curl/upload/file/posix.tmpl.sh rename to curl/upload/posix.tmpl.sh index 59ee342..4c971b9 100644 --- a/curl/upload/file/posix.tmpl.sh +++ b/curl/upload/posix.tmpl.sh @@ -8,9 +8,10 @@ CURL="{{curl}}" SRC="{{src}}" DST="{{dst}}" URL="{{url}}" +UPLOAD_DATA="{{upload_data}}" RETRY="{{retry}}" RETRY_DELAY="{{retry_delay}}" -readonly CURL SRC DST URL RETRY RETRY_DELAY +readonly CURL SRC DST URL RETRY RETRY_DELAY UPLOAD_DATA # Runfiles RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" @@ -62,14 +63,25 @@ readonly ENDPOINT DIRECTORY DESTINATION COMPOSED="${ENDPOINT}${DIRECTORY}${DESTINATION}" readonly COMPOSED -printf >&2 "Uploading: %s to %s\n" "${SRC}" "${COMPOSED}" +upload() { -# Do the upload -"${RUNFILES}/${CURL}" \ - --netrc \ - --location \ - --progress-bar \ - --retry "${RETRY}" \ - --retry-delay "${RETRY_DELAY}" \ - --upload-file "${RUNFILES}/${SRC}" \ - "${COMPOSED}" + printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" + + # Do the upload + "${RUNFILES}/${CURL}" \ + --netrc \ + --location \ + --progress-bar \ + --retry "${RETRY}" \ + --retry-delay "${RETRY_DELAY}" \ + --upload-file "${RUNFILES}/${1}" \ + "${2}" +} + +if test -n ${UPLOAD_DATA} && test -f "${RUNFILES}/${UPLOAD_DATA}"; then + while IFS= read -r line; do + upload "${line}" "${COMPOSED}/${line##*/}" + done < "${RUNFILES}/${UPLOAD_DATA}" +else + upload "${SRC}" "${COMPOSED}" +fi diff --git a/curl/upload/providers.bzl b/curl/upload/providers.bzl new file mode 100644 index 0000000..0f37a7e --- /dev/null +++ b/curl/upload/providers.bzl @@ -0,0 +1,8 @@ +visibility("public") + +UploadFilesInfo = provider( + doc = "Detailed information for files to be uploaded.", + fields = { + "files" : "A depset of files to upload.", + }, +) -- GitLab From 873c9d44b9d24c6a432633f2ebac9ebac989abf4 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Wed, 12 Jun 2024 13:04:27 +0200 Subject: [PATCH 2/8] chore: test uploading files Add tests to verify uloading bunch of files. --- e2e/mock/rule.bzl | 51 ++++++++++++++++++++++++++++++ e2e/upload/files/BUILD.bazel | 60 ++++++++++++++++++++++++++++++++++++ e2e/upload/files/fixture.txt | 40 ++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 e2e/mock/rule.bzl create mode 100644 e2e/upload/files/BUILD.bazel create mode 100644 e2e/upload/files/fixture.txt diff --git a/e2e/mock/rule.bzl b/e2e/mock/rule.bzl new file mode 100644 index 0000000..716c46c --- /dev/null +++ b/e2e/mock/rule.bzl @@ -0,0 +1,51 @@ +load("@rules_curl//curl/upload:providers.bzl", "UploadFilesInfo") + +DOC = """Mock rule with output `UploadFilesInfo` provider. + +```py +mock_files_to_upload( + name = "mock_files", + outs = [ + "file_1.out", + "file_2.out", + ], + deps = [ + "my_dep", + ], +) +``` +""" + +ATTRS = { + "outs": attr.string_list( + default = [], + mandatory = True, + ), + "deps": attr.label_list(allow_files = True), +} + +def implementation(ctx): + outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] + + for f in outs: + ctx.actions.write(f, "hello test", is_executable=False) + + files = depset(outs) + + return [ + DefaultInfo( + files = files, + ), + UploadFilesInfo( + files = depset( + direct = outs, + transitive = [dep[UploadFilesInfo].files for dep in ctx.attr.deps if UploadFilesInfo in dep], + ), + ), + ] + +mock_files_to_upload = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, +) diff --git a/e2e/upload/files/BUILD.bazel b/e2e/upload/files/BUILD.bazel new file mode 100644 index 0000000..229a6a5 --- /dev/null +++ b/e2e/upload/files/BUILD.bazel @@ -0,0 +1,60 @@ +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("@rules_curl//curl/upload/files:defs.bzl", "curl_upload_files") +load("//mock:rule.bzl", "mock_files_to_upload") + +genrule( + name = "data", + testonly = True, + outs = [ + "data.out", + ], + cmd = "echo 'Hello data\n' > $@", +) + +mock_files_to_upload( + name = "file_pack_1", + outs = [ + "file_1.out", + "file_2.out", + ], + testonly = True, +) + +mock_files_to_upload( + name = "file_pack_2", + outs = [ + "file_A.txt", + "file_B.txt", + ], + deps = [ + ":file_pack_1", + ":data", + ], + testonly = True, +) + +curl_upload_files( + name = "upload_files", + testonly = True, + srcs = [ + ":file_pack_2", + ], + dst = "destination/directory/to/upload", + url = "https://test.case", +) + +genrule( + name = "execute", + testonly = True, + outs = [ + "upload_files.out", + ], + cmd = "./$(location :upload_files) > $@", + tools = [":upload_files"], +) + +diff_test( + name = "test", + file1 = ":fixture.txt", + file2 = ":execute", +) diff --git a/e2e/upload/files/fixture.txt b/e2e/upload/files/fixture.txt new file mode 100644 index 0000000..695b83b --- /dev/null +++ b/e2e/upload/files/fixture.txt @@ -0,0 +1,40 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_1.out +https://test.case/destination/directory/to/upload/file_1.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_2.out +https://test.case/destination/directory/to/upload/file_2.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_A.txt +https://test.case/destination/directory/to/upload/file_A.txt +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_B.txt +https://test.case/destination/directory/to/upload/file_B.txt -- GitLab From 7eb6a7e468cb9abe7a065863a08fa2ce4a85618a Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 13 Jun 2024 08:55:59 +0200 Subject: [PATCH 3/8] chore: use `rules_diff` `diff_test` from `@bazel_skylib` uses non-hermetic `diff` tool from the host. Move to hermetic `diff` tool. --- e2e/MODULE.bazel | 1 + e2e/MODULE.bazel.lock | 32 +++++++++++++++++++++++++++++++- e2e/upload/file/BUILD.bazel | 9 +++++---- e2e/upload/files/BUILD.bazel | 9 +++++---- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index 03a276e..cec2f49 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -7,6 +7,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") +bazel_dep(name = "rules_diff", version = "1.0.0-beta.3") bazel_dep(name = "rules_curl") local_path_override( module_name = "rules_curl", diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index fca858b..933c582 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -39,6 +39,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/MODULE.bazel": "4bcae1c5e3c7fa1169f9940f548f7b8b3316944b4367771b168f925b7a9ee74e", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/source.json": "b5db3fcd469061f2051188da97345162f294fc59e7fdf477beb306dbe950566a", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -117,7 +119,7 @@ "@@toolchain_utils~//toolchain/export:defs.bzl%toolchain_export": { "general": { "bzlTransitiveDigest": "n2fd+/jiAv/nvvi2WUR+VQxI3aTZGNuhuX/NuFhw5fM=", - "usagesDigest": "A2BbbNuBP0QFNqjCVowoaEOrolUDkTAis6lyamxEXLE=", + "usagesDigest": "OT9pdXtWdZUfYUSpReUZ6+fci56uJ5AVYR22QqnoBYI=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -374,6 +376,13 @@ "target": "@@ape~~_repo_rules~seq//:seq" } }, + "diff": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-diff//:ape-diff" + } + }, "ape-printenv": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", @@ -549,6 +558,13 @@ "target": "@@ape~~_repo_rules~qjs//:qjs" } }, + "cmp": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-cmp//:ape-cmp" + } + }, "ape-df": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", @@ -738,6 +754,20 @@ "target": "@@ape~~_repo_rules~nohup//:nohup" } }, + "sdiff": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-sdiff//:ape-sdiff" + } + }, + "diff3": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-diff3//:ape-diff3" + } + }, "ape-basename": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", diff --git a/e2e/upload/file/BUILD.bazel b/e2e/upload/file/BUILD.bazel index 6259557..6826b08 100644 --- a/e2e/upload/file/BUILD.bazel +++ b/e2e/upload/file/BUILD.bazel @@ -1,5 +1,5 @@ -load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") +load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") curl_upload_file( name = "upload", @@ -19,8 +19,9 @@ genrule( tools = [":upload"], ) -diff_test( +diff_file_test( name = "test", - file1 = ":fixture.txt", - file2 = ":execute", + size = "small", + a = ":fixture.txt", + b = ":execute", ) diff --git a/e2e/upload/files/BUILD.bazel b/e2e/upload/files/BUILD.bazel index 229a6a5..c812c61 100644 --- a/e2e/upload/files/BUILD.bazel +++ b/e2e/upload/files/BUILD.bazel @@ -1,5 +1,5 @@ -load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@rules_curl//curl/upload/files:defs.bzl", "curl_upload_files") +load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") load("//mock:rule.bzl", "mock_files_to_upload") genrule( @@ -53,8 +53,9 @@ genrule( tools = [":upload_files"], ) -diff_test( +diff_file_test( name = "test", - file1 = ":fixture.txt", - file2 = ":execute", + size = "small", + a = ":fixture.txt", + b = ":execute", ) -- GitLab From 22174f7b2e9c1e04f3e91e873ff4fa832d07bdfb Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 13 Jun 2024 12:12:58 +0200 Subject: [PATCH 4/8] chore: new providers Change introduces new providers: - `CurlUploadManifestInfo` - `CurlUploadManifestsInfo` Those are replacing `UploadFilesInfo` provider. Sources for `curl_upload_files` rule have to provide either `CurlUploadManifestInfo` or `CurlUploadManifestsInfo`. --- MODULE.bazel | 1 + MODULE.bazel.lock | 1 + curl/upload/ManifestInfo.bzl | 50 +++++++++++++++++++++++++++++++++++ curl/upload/ManifestsInfo.bzl | 34 ++++++++++++++++++++++++ curl/upload/files/rule.bzl | 41 +++++++++++++++++++--------- curl/upload/posix.tmpl.sh | 6 ++--- curl/upload/providers.bzl | 8 ------ 7 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 curl/upload/ManifestInfo.bzl create mode 100644 curl/upload/ManifestsInfo.bzl delete mode 100644 curl/upload/providers.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 311e5c0..31e307f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,6 +5,7 @@ module( ], ) +bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") bazel_dep(name = "ape", version = "1.0.0-beta.6") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 197277a..6fbba1d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -15,6 +15,7 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", diff --git a/curl/upload/ManifestInfo.bzl b/curl/upload/ManifestInfo.bzl new file mode 100644 index 0000000..17edaa6 --- /dev/null +++ b/curl/upload/ManifestInfo.bzl @@ -0,0 +1,50 @@ +load("@bazel_skylib//lib:types.bzl", "types") + +visibility("public") + +def init(file, url = "{href}/{dst}/{basename}"): + """ + Initializes a `CurlUploadManifestInfo` provider. + + To be used with `curl_upload_manifests` + + Args: + file: The file to upload. + url: A Starlark string template for the URL, which can include: + - href: the provided + - dst: the directory to upload into + - path: the full path of the source file + - dirname: the directory name of the source file + - basename: the basename of the source file + - stem: the stem of the source file + - extension: the extension of the source file + Defaults to `{href}/{dst}/{basename}` + + Returns: + A mapping of keywords for the `curl_upload_manifest_info` raw constructor. + """ + if type(file) != "File": + fail("`CurlUploadManifestInfo.file` must be a `file`: {}".format(file)) + + if file.is_directory: + fail("`CurlUploadManifestInfo.file` must not be a directory: {}".format(file)) + + if not types.is_string(url): + fail("`CurlUploadManifestInfo.url` must be a `str`: {}".format(url)) + + return { + "file": file, + "url": url, + } + +CurlUploadManifestInfo, curl_upload_manifest_info = provider( + "A file to upload with cURL.", + fields = ["file", "url"], + init = init, +) + +# Provide some convenience imports +ManifestInfo = CurlUploadManifestInfo +manifest_info = curl_upload_manifest_info +Info = CurlUploadManifestInfo +info = curl_upload_manifest_info diff --git a/curl/upload/ManifestsInfo.bzl b/curl/upload/ManifestsInfo.bzl new file mode 100644 index 0000000..9bf15e7 --- /dev/null +++ b/curl/upload/ManifestsInfo.bzl @@ -0,0 +1,34 @@ +load("@bazel_skylib//lib:types.bzl", "types") + +visibility("public") + +def init(manifests): + """ + Initializes a `CurlUploadManifestsInfo` provider. + + To be used with `curl_upload_manifests` + + Args: + manifests: The dependency set of `CurlUploadManifestInfo`s + + Returns: + A mapping of keywords for the `curl_upload_manifests_info` raw constructor. + """ + if not types.is_depset(manifests): + fail("`CurlUploadManifestsInfo.url` must be a `depset`: {}".format(manifests)) + + return { + "manifests": manifests, + } + +CurlUploadManifestsInfo, curl_upload_manifests_info = provider( + "Files to upload with cURL.", + fields = ["manifests"], + init = init, +) + +# Provide some convenience imports +ManifestsInfo = CurlUploadManifestsInfo +manifests_info = curl_upload_manifests_info +Info = CurlUploadManifestsInfo +info = curl_upload_manifests_info diff --git a/curl/upload/files/rule.bzl b/curl/upload/files/rule.bzl index a9572de..7c80fdb 100644 --- a/curl/upload/files/rule.bzl +++ b/curl/upload/files/rule.bzl @@ -1,14 +1,15 @@ -load("//curl/upload:providers.bzl", "UploadFilesInfo") +load("//curl/upload:ManifestInfo.bzl", "ManifestInfo") +load("//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") visibility("//curl/...") DOC = """Upload bunch of files to a URL endpoint with cURL. -The `srcs` must provide `UploadFilesInfo`. +The `srcs` must provide `ManifestInfo` or `ManifestsInfo`. ```py file( - name = "upload_file", + name = "upload_files", srcs = [ ":data", ], @@ -22,22 +23,25 @@ ATTRS = { "srcs": attr.label_list( doc = "Files to be uploaded.", mandatory = True, - providers = [UploadFilesInfo], + providers = [ + [ManifestInfo], + [ManifestsInfo], + ], ), "dst": attr.string( doc = "The destination directory to upload.", default = "", ), "url": attr.string( - doc = "URL endpoint for file upload.", + doc = "URL endpoint for files to upload.", mandatory = True, ), "retry": attr.int( - doc = "The number of retry attempts.", + doc = "The number of retry attempts for every file.", default = 3, ), "retry_delay": attr.int( - doc = "The seconds to wait before attempting a upload retry.", + doc = "The seconds to wait before attempting an upload retry.", default = 1, ), "template": attr.label( @@ -52,12 +56,25 @@ def implementation(ctx): content = "" upload_data = ctx.actions.declare_file("upload_data.txt") - upload = depset(transitive = [src[UploadFilesInfo].files for src in ctx.attr.srcs]) + manifests = depset( + direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], + transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], + ) - for f in upload.to_list(): - content += "{}\n".format(f.short_path) + for m in manifests.to_list(): + _url = m.url.format( + path = m.file.path, + dirname = m.file.dirname, + basename = m.file.basename, + extension = m.file.extension, + href = ctx.attr.url.rstrip("/"), + dst = ctx.attr.dst.rstrip("/"), + stem = m.file.basename.rstrip(".{}".format(m.file.extension)), + ) + _file = m.file.short_path + content += "{} {}\n".format(_file, _url) - ctx.actions.write(upload_data, content, is_executable=False) + ctx.actions.write(upload_data, content, is_executable = False) executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) @@ -78,7 +95,7 @@ def implementation(ctx): ) files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, upload_data] + ctx.files.srcs + upload.to_list()) + runfiles = ctx.runfiles([curl.executable, upload_data] + ctx.files.srcs + [m.file for m in manifests.to_list()]) runfiles = runfiles.merge(curl.default.default_runfiles) return DefaultInfo( diff --git a/curl/upload/posix.tmpl.sh b/curl/upload/posix.tmpl.sh index 4c971b9..428f72c 100644 --- a/curl/upload/posix.tmpl.sh +++ b/curl/upload/posix.tmpl.sh @@ -79,9 +79,9 @@ upload() { } if test -n ${UPLOAD_DATA} && test -f "${RUNFILES}/${UPLOAD_DATA}"; then - while IFS= read -r line; do - upload "${line}" "${COMPOSED}/${line##*/}" - done < "${RUNFILES}/${UPLOAD_DATA}" + while read -r src dst; do + upload "${src}" "${dst}" + done <"${RUNFILES}/${UPLOAD_DATA}" else upload "${SRC}" "${COMPOSED}" fi diff --git a/curl/upload/providers.bzl b/curl/upload/providers.bzl deleted file mode 100644 index 0f37a7e..0000000 --- a/curl/upload/providers.bzl +++ /dev/null @@ -1,8 +0,0 @@ -visibility("public") - -UploadFilesInfo = provider( - doc = "Detailed information for files to be uploaded.", - fields = { - "files" : "A depset of files to upload.", - }, -) -- GitLab From fcbb508c05ff0c565d159536e737ef942181fe8b Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 13 Jun 2024 12:18:20 +0200 Subject: [PATCH 5/8] chore: adjust tests Test rule uploading multiple files who uses `CurlUploadManifestInfo` and `CurlUploadManifestsInfo` providers. --- e2e/mock/manifest/BUILD.bazel | 0 e2e/mock/manifest/rule.bzl | 30 +++++++++++++++++++ e2e/mock/manifests/BUILD.bazel | 0 e2e/mock/manifests/rule.bzl | 54 ++++++++++++++++++++++++++++++++++ e2e/mock/rule.bzl | 51 -------------------------------- e2e/upload/files/BUILD.bazel | 23 +++++++++++---- e2e/upload/files/fixture.txt | 20 +++++++++++++ 7 files changed, 121 insertions(+), 57 deletions(-) create mode 100644 e2e/mock/manifest/BUILD.bazel create mode 100644 e2e/mock/manifest/rule.bzl create mode 100644 e2e/mock/manifests/BUILD.bazel create mode 100644 e2e/mock/manifests/rule.bzl delete mode 100644 e2e/mock/rule.bzl diff --git a/e2e/mock/manifest/BUILD.bazel b/e2e/mock/manifest/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl new file mode 100644 index 0000000..269e4be --- /dev/null +++ b/e2e/mock/manifest/rule.bzl @@ -0,0 +1,30 @@ +load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo") + +DOC = """Mock rule with output `ManifestInfo` provider. + +```py +mock_manifest( + name = "mock_file", +) +``` +""" + +def _impl(ctx): + out = ctx.actions.declare_file("{}.out".format(ctx.attr.name)) + + ctx.actions.write(out, "hello test", is_executable = False) + + return [ + DefaultInfo( + files = depset([out]), + ), + ManifestInfo( + file = out, + url = "{href}/{basename}", + ), + ] + +mock_manifest = rule( + doc = DOC, + implementation = _impl, +) diff --git a/e2e/mock/manifests/BUILD.bazel b/e2e/mock/manifests/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl new file mode 100644 index 0000000..d15466a --- /dev/null +++ b/e2e/mock/manifests/rule.bzl @@ -0,0 +1,54 @@ +load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info") +load("@rules_curl//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") + +DOC = """Mock rule with output `ManifestsInfo` provider. + +```py +mock_manifests( + name = "mock_files", + outs = [ + "file_1.out", + "file_2.out", + ], + deps = [ + "my_dep", + ], +) +``` +""" + +ATTRS = { + "outs": attr.string_list( + default = [], + mandatory = True, + ), + "deps": attr.label_list(allow_files = True), +} + +def _impl(ctx): + outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] + + for f in outs: + ctx.actions.write(f, "hello test", is_executable = False) + + files = depset(outs) + + manifests = [manifest_info(file = o, url = "{href}/{dst}/{stem}.{extension}") for o in outs] + manifests.extend([dep[ManifestInfo] for dep in ctx.attr.deps if ManifestInfo in dep]) + + transitive = [dep[ManifestsInfo].manifests for dep in ctx.attr.deps if ManifestsInfo in dep] + + return [ + DefaultInfo( + files = files, + ), + ManifestsInfo( + manifests = depset(direct = manifests, transitive = transitive), + ), + ] + +mock_manifests = rule( + doc = DOC, + attrs = ATTRS, + implementation = _impl, +) diff --git a/e2e/mock/rule.bzl b/e2e/mock/rule.bzl deleted file mode 100644 index 716c46c..0000000 --- a/e2e/mock/rule.bzl +++ /dev/null @@ -1,51 +0,0 @@ -load("@rules_curl//curl/upload:providers.bzl", "UploadFilesInfo") - -DOC = """Mock rule with output `UploadFilesInfo` provider. - -```py -mock_files_to_upload( - name = "mock_files", - outs = [ - "file_1.out", - "file_2.out", - ], - deps = [ - "my_dep", - ], -) -``` -""" - -ATTRS = { - "outs": attr.string_list( - default = [], - mandatory = True, - ), - "deps": attr.label_list(allow_files = True), -} - -def implementation(ctx): - outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] - - for f in outs: - ctx.actions.write(f, "hello test", is_executable=False) - - files = depset(outs) - - return [ - DefaultInfo( - files = files, - ), - UploadFilesInfo( - files = depset( - direct = outs, - transitive = [dep[UploadFilesInfo].files for dep in ctx.attr.deps if UploadFilesInfo in dep], - ), - ), - ] - -mock_files_to_upload = rule( - doc = DOC, - attrs = ATTRS, - implementation = implementation, -) diff --git a/e2e/upload/files/BUILD.bazel b/e2e/upload/files/BUILD.bazel index c812c61..cf65214 100644 --- a/e2e/upload/files/BUILD.bazel +++ b/e2e/upload/files/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_curl//curl/upload/files:defs.bzl", "curl_upload_files") load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("//mock:rule.bzl", "mock_files_to_upload") +load("//mock/manifest:rule.bzl", "mock_manifest") +load("//mock/manifests:rule.bzl", "mock_manifests") genrule( name = "data", @@ -11,26 +12,35 @@ genrule( cmd = "echo 'Hello data\n' > $@", ) -mock_files_to_upload( +mock_manifest( + name = "fixture_1", +) + +mock_manifest( + name = "fixture_2", +) + +mock_manifests( name = "file_pack_1", + testonly = True, outs = [ "file_1.out", "file_2.out", ], - testonly = True, ) -mock_files_to_upload( +mock_manifests( name = "file_pack_2", + testonly = True, outs = [ "file_A.txt", "file_B.txt", ], deps = [ - ":file_pack_1", ":data", + ":file_pack_1", + ":fixture_2", ], - testonly = True, ) curl_upload_files( @@ -38,6 +48,7 @@ curl_upload_files( testonly = True, srcs = [ ":file_pack_2", + ":fixture_1", ], dst = "destination/directory/to/upload", url = "https://test.case", diff --git a/e2e/upload/files/fixture.txt b/e2e/upload/files/fixture.txt index 695b83b..f6e85f5 100644 --- a/e2e/upload/files/fixture.txt +++ b/e2e/upload/files/fixture.txt @@ -38,3 +38,23 @@ https://test.case/destination/directory/to/upload/file_A.txt --upload-file file_B.txt https://test.case/destination/directory/to/upload/file_B.txt +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +fixture_2.out +https://test.case/fixture_2.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +fixture_1.out +https://test.case/fixture_1.out -- GitLab From 70150c4c001551000f611dce0756a7f62d118a66 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 13 Jun 2024 15:15:10 +0200 Subject: [PATCH 6/8] chore: unnecessary `to_list()` call Reduce calling of `depset.to_list()` twice. --- curl/upload/files/rule.bzl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/curl/upload/files/rule.bzl b/curl/upload/files/rule.bzl index 7c80fdb..635f810 100644 --- a/curl/upload/files/rule.bzl +++ b/curl/upload/files/rule.bzl @@ -61,7 +61,9 @@ def implementation(ctx): transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], ) - for m in manifests.to_list(): + manifest_files = manifests.to_list() + + for m in manifest_files: _url = m.url.format( path = m.file.path, dirname = m.file.dirname, @@ -95,7 +97,7 @@ def implementation(ctx): ) files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, upload_data] + ctx.files.srcs + [m.file for m in manifests.to_list()]) + runfiles = ctx.runfiles([curl.executable, upload_data, test_data] + ctx.files.srcs + [m.file for m in manifest_files]) runfiles = runfiles.merge(curl.default.default_runfiles) return DefaultInfo( -- GitLab From 167630d3ecfcd9e273f52b408258ecb19fb25d3b Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 13 Jun 2024 16:00:53 +0200 Subject: [PATCH 7/8] chore: defer depset expansion to execution phase --- curl/upload/files/rule.bzl | 21 +++++++++++---------- curl/upload/posix.tmpl.sh | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/curl/upload/files/rule.bzl b/curl/upload/files/rule.bzl index 635f810..eb0f9c9 100644 --- a/curl/upload/files/rule.bzl +++ b/curl/upload/files/rule.bzl @@ -54,29 +54,30 @@ ATTRS = { def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] - content = "" - upload_data = ctx.actions.declare_file("upload_data.txt") + upload_data = ctx.actions.declare_file("upload.csv") manifests = depset( direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], ) - manifest_files = manifests.to_list() + href = ctx.attr.url.rstrip("/") + dst = ctx.attr.dst.rstrip("/") - for m in manifest_files: + def _to_string(m): _url = m.url.format( path = m.file.path, dirname = m.file.dirname, basename = m.file.basename, extension = m.file.extension, - href = ctx.attr.url.rstrip("/"), - dst = ctx.attr.dst.rstrip("/"), + href = href, + dst = dst, stem = m.file.basename.rstrip(".{}".format(m.file.extension)), ) - _file = m.file.short_path - content += "{} {}\n".format(_file, _url) + return "{},{}".format(m.file.short_path, _url) - ctx.actions.write(upload_data, content, is_executable = False) + args = ctx.actions.args() + args.add_all(manifests, map_each = _to_string, allow_closure = True) + ctx.actions.write(upload_data, args, is_executable = False) executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) @@ -97,7 +98,7 @@ def implementation(ctx): ) files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, upload_data, test_data] + ctx.files.srcs + [m.file for m in manifest_files]) + runfiles = ctx.runfiles([curl.executable, upload_data] + ctx.files.srcs + [m.file for m in manifests.to_list()]) runfiles = runfiles.merge(curl.default.default_runfiles) return DefaultInfo( diff --git a/curl/upload/posix.tmpl.sh b/curl/upload/posix.tmpl.sh index 428f72c..6e0d43e 100644 --- a/curl/upload/posix.tmpl.sh +++ b/curl/upload/posix.tmpl.sh @@ -79,8 +79,8 @@ upload() { } if test -n ${UPLOAD_DATA} && test -f "${RUNFILES}/${UPLOAD_DATA}"; then - while read -r src dst; do - upload "${src}" "${dst}" + while IFS= read -r line; do + upload "${line%,*}" "${line#*,}" done <"${RUNFILES}/${UPLOAD_DATA}" else upload "${SRC}" "${COMPOSED}" -- GitLab From e6785d1f3d51e54bbf3989457abd406608069b87 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 14 Jun 2024 11:32:11 +0200 Subject: [PATCH 8/8] chore: rule providers Specify providers returned by the rule implementation. --- e2e/mock/manifest/rule.bzl | 3 +++ e2e/mock/manifests/rule.bzl | 3 +++ 2 files changed, 6 insertions(+) diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl index 269e4be..79e997e 100644 --- a/e2e/mock/manifest/rule.bzl +++ b/e2e/mock/manifest/rule.bzl @@ -27,4 +27,7 @@ def _impl(ctx): mock_manifest = rule( doc = DOC, implementation = _impl, + provides = [ + ManifestInfo, + ], ) diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl index d15466a..e422988 100644 --- a/e2e/mock/manifests/rule.bzl +++ b/e2e/mock/manifests/rule.bzl @@ -51,4 +51,7 @@ mock_manifests = rule( doc = DOC, attrs = ATTRS, implementation = _impl, + provides = [ + ManifestsInfo, + ], ) -- GitLab