From 9f2fa29119cd0cbc50996b10829135848fd7e712 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 20 Jun 2024 16:21:05 +0200 Subject: [PATCH 01/12] chore: add manifest providers --- MODULE.bazel | 1 + MODULE.bazel.lock | 1 + curl/upload/BUILD.bazel | 0 curl/upload/ManifestInfo.bzl | 53 +++++++++++++++++++++++++++++++++++ curl/upload/ManifestsInfo.bzl | 34 ++++++++++++++++++++++ 5 files changed, 89 insertions(+) create mode 100644 curl/upload/BUILD.bazel create mode 100644 curl/upload/ManifestInfo.bzl create mode 100644 curl/upload/ManifestsInfo.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 311e5c0..31e307f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,6 +5,7 @@ module( ], ) +bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") bazel_dep(name = "ape", version = "1.0.0-beta.6") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 197277a..6fbba1d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -15,6 +15,7 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", diff --git a/curl/upload/BUILD.bazel b/curl/upload/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/curl/upload/ManifestInfo.bzl b/curl/upload/ManifestInfo.bzl new file mode 100644 index 0000000..175f565 --- /dev/null +++ b/curl/upload/ManifestInfo.bzl @@ -0,0 +1,53 @@ +load("@bazel_skylib//lib:types.bzl", "types") + +visibility("public") + +def init(file, url): + """ + Initializes a `CurlUploadManifestInfo` provider. + + To be used with `curl_upload_manifests` + + Args: + file: The file to upload. + url: A Starlark string template for the URL, which can include: + - {{Href}}: the provided URL + - {{.Url.Hostname}}: the domain name of the URL + - {{.Url.Port}}: the port number of the URL + - {{.Url.Protocol}}: the protocol scheme of the URL + - {{.Url.Pathname}}: a location in a hierarchical structure + - {{.File.Path}}: the full path of the source file + - {{.File.Basename}}: the basename of the source file + - {{.File.Extension}}: the extension of the source file + + Returns: + A mapping of keywords for the `curl_upload_manifest_info` raw constructor. + """ + if type(file) != "File": + fail("`CurlUploadManifestInfo.file` must be a `file`: {}".format(file)) + + if file.is_directory: + fail("`CurlUploadManifestInfo.file` must not be a directory: {}".format(file)) + + if not types.is_string(url): + fail("`CurlUploadManifestInfo.url` must be a `str`: {}".format(url)) + + if url.find(",") != -1: + fail("`CurlUploadManifestInfo.url` must not have comma `,` sign: {}".format(url)) + + return { + "file": file, + "url": url, + } + +CurlUploadManifestInfo, curl_upload_manifest_info = provider( + "A file to upload with cURL.", + fields = ["file", "url"], + init = init, +) + +# Provide some convenience imports +ManifestInfo = CurlUploadManifestInfo +manifest_info = curl_upload_manifest_info +Info = CurlUploadManifestInfo +info = curl_upload_manifest_info diff --git a/curl/upload/ManifestsInfo.bzl b/curl/upload/ManifestsInfo.bzl new file mode 100644 index 0000000..9bf15e7 --- /dev/null +++ b/curl/upload/ManifestsInfo.bzl @@ -0,0 +1,34 @@ +load("@bazel_skylib//lib:types.bzl", "types") + +visibility("public") + +def init(manifests): + """ + Initializes a `CurlUploadManifestsInfo` provider. + + To be used with `curl_upload_manifests` + + Args: + manifests: The dependency set of `CurlUploadManifestInfo`s + + Returns: + A mapping of keywords for the `curl_upload_manifests_info` raw constructor. + """ + if not types.is_depset(manifests): + fail("`CurlUploadManifestsInfo.url` must be a `depset`: {}".format(manifests)) + + return { + "manifests": manifests, + } + +CurlUploadManifestsInfo, curl_upload_manifests_info = provider( + "Files to upload with cURL.", + fields = ["manifests"], + init = init, +) + +# Provide some convenience imports +ManifestsInfo = CurlUploadManifestsInfo +manifests_info = curl_upload_manifests_info +Info = CurlUploadManifestsInfo +info = curl_upload_manifests_info -- GitLab From 1c36f3d02a84ff733c6e5fbc06a23c8ccf233f4b Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 20 Jun 2024 16:24:24 +0200 Subject: [PATCH 02/12] chore: templating tool Go `template` tool for templating URL. --- MODULE.bazel | 1 + MODULE.bazel.lock | 204 ++++++++++++++++++++++++++++++++++++++ curl/template/BUILD.bazel | 7 ++ curl/template/template.go | 104 +++++++++++++++++++ e2e/MODULE.bazel.lock | 204 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 520 insertions(+) create mode 100644 curl/template/BUILD.bazel create mode 100644 curl/template/template.go diff --git a/MODULE.bazel b/MODULE.bazel index 31e307f..5614dd7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,6 +8,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") bazel_dep(name = "ape", version = "1.0.0-beta.6") +bazel_dep(name = "rules_go", version = "0.46.0") export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export") use_repo(export, "ape-curl") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 6fbba1d..68eddf6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -9,9 +9,12 @@ "https://bcr.bazel.build/modules/ape/1.0.0-beta.6/source.json": "a2dda258a2985f703b359d4e287863489314078675e77c4afbf57122324d7c8a", "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", + "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", @@ -22,6 +25,10 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -33,12 +40,17 @@ "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -113,6 +125,198 @@ }, "recordedRepoMappingEntries": [] } + }, + "@@rules_go~//go:extensions.bzl%go_sdk": { + "os:linux,arch:amd64": { + "bzlTransitiveDigest": "ngSfLTe8hQiidDeEb6YUqr77yNHrYln0spUfoRZ8O5I=", + "usagesDigest": "eBPCRLKeBr4iU55b1S2aLCYk6yMHyuLgphY9KhjbwDU=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "io_bazel_rules_nogo": { + "bzlFile": "@@rules_go~//go/private:nogo.bzl", + "ruleClassName": "go_register_nogo", + "attributes": { + "nogo": "@io_bazel_rules_go//:default_nogo", + "includes": [ + "'@@//:__subpackages__'" + ], + "excludes": [] + } + }, + "rules_go__download_0_windows_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "rules_go__download_0_linux_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_default_sdk": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "experiments": [], + "patches": [], + "patch_strip": 0, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1", + "strip_prefix": "go" + } + }, + "rules_go__download_0_darwin_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_host_compatible_sdk_label": { + "bzlFile": "@@rules_go~//go/private:extensions.bzl", + "ruleClassName": "host_compatible_toolchain", + "attributes": { + "toolchain": "@go_default_sdk//:ROOT" + } + }, + "rules_go__download_0_darwin_amd64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_toolchains": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_multiple_toolchains", + "attributes": { + "prefixes": [ + "_0000_go_default_sdk_", + "_0001_rules_go__download_0_darwin_amd64_", + "_0002_rules_go__download_0_darwin_arm64_", + "_0003_rules_go__download_0_linux_arm64_", + "_0004_rules_go__download_0_windows_amd64_", + "_0005_rules_go__download_0_windows_arm64_" + ], + "geese": [ + "", + "darwin", + "darwin", + "linux", + "windows", + "windows" + ], + "goarchs": [ + "", + "amd64", + "arm64", + "arm64", + "amd64", + "arm64" + ], + "sdk_repos": [ + "go_default_sdk", + "rules_go__download_0_darwin_amd64", + "rules_go__download_0_darwin_arm64", + "rules_go__download_0_linux_arm64", + "rules_go__download_0_windows_amd64", + "rules_go__download_0_windows_arm64" + ], + "sdk_types": [ + "remote", + "remote", + "remote", + "remote", + "remote", + "remote" + ], + "sdk_versions": [ + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1" + ] + } + }, + "rules_go__download_0_windows_amd64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "bazel_features~", + "bazel_features_globals", + "bazel_features~~version_extension~bazel_features_globals" + ], + [ + "bazel_features~", + "bazel_features_version", + "bazel_features~~version_extension~bazel_features_version" + ], + [ + "rules_go~", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_go~", + "io_bazel_rules_go", + "rules_go~" + ], + [ + "rules_go~", + "io_bazel_rules_go_bazel_features", + "bazel_features~" + ] + ] + } } } } diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel new file mode 100644 index 0000000..fc9c00b --- /dev/null +++ b/curl/template/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_go//go:def.bzl", "go_binary") + +go_binary( + name = "template", + srcs = ["template.go"], + visibility = ["//curl/upload:__subpackages__"], +) diff --git a/curl/template/template.go b/curl/template/template.go new file mode 100644 index 0000000..c66628e --- /dev/null +++ b/curl/template/template.go @@ -0,0 +1,104 @@ +package main + +import ( + "flag" + "html/template" + "log" + "net/url" + "os" + "path/filepath" +) + +type StringInfo string + +type FileInfo struct { + Path StringInfo + Basename StringInfo + Extension StringInfo +} + +type UrlInfo struct { + Hostname StringInfo + Port StringInfo + Pathname StringInfo + Protocol StringInfo +} + +func (u *UrlInfo) Href() string { + return u.Protocol.String() + "//" + u.Hostname.String() + u.Port.String() + u.Pathname.String() +} + +type ManifestInfo struct { + Url UrlInfo + File FileInfo +} + +func (f *FileInfo) String() string { + return string(f.Path) +} + +func (f *FileInfo) Set(value string) error { + cleaned := filepath.Clean(value) + f.Path = StringInfo(filepath.ToSlash(cleaned)) + f.Basename = StringInfo(filepath.ToSlash(filepath.Base(cleaned))) + f.Extension = StringInfo(filepath.ToSlash(filepath.Ext(cleaned))) + return nil +} + +func (u *UrlInfo) String() string { + return u.Href() +} + +func (u *UrlInfo) Set(value string) error { + parsed, err := url.Parse(value) + if err != nil { + return err + } + u.Hostname = StringInfo(parsed.Hostname()) + u.Port = StringInfo(parsed.Port()) + u.Pathname = StringInfo(parsed.Path) + u.Protocol = StringInfo(parsed.Scheme + ":") + return err +} + +func (s *StringInfo) String() string { + return string(*s) +} + +func (s *StringInfo) Set(value string) error { + log.Fatal(value) + *s = StringInfo(value) + return nil +} + +func main() { + + manifest := ManifestInfo{ + Url: UrlInfo{}, + File: FileInfo{}, + } + + flag.Var(&manifest.File, "file", "The file path to use for templating") + flag.Var(&manifest.File.Basename, "basename", "The basename of the source file") + flag.Var(&manifest.File.Extension, "extension", "The extension of the source file") + flag.Var(&manifest.Url, "url", "URL to use for templating") + flag.Var(&manifest.Url.Hostname, "hostname", "The domain name of the URL") + flag.Var(&manifest.Url.Port, "port", "The port number of the URL") + flag.Var(&manifest.Url.Pathname, "pathname", "A location in a hierachical structure") + flag.Var(&manifest.Url.Protocol, "scheme", "The scheme for the URL") + t := flag.String("template", "{{Href}}/{{.File.Path}}", "The Go template to render") + + flag.Parse() + + var funcs = template.FuncMap{"Href": manifest.Url.Href} + + tmpl, err := template.New("url").Funcs(funcs).Parse(*t) + if err != nil { + log.Fatal(err) + } + + err = tmpl.Execute(os.Stdout, manifest) + if err != nil { + log.Fatal(err) + } +} diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index fca858b..50ef842 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -9,9 +9,12 @@ "https://bcr.bazel.build/modules/ape/1.0.0-beta.6/source.json": "a2dda258a2985f703b359d4e287863489314078675e77c4afbf57122324d7c8a", "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", + "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", @@ -22,6 +25,10 @@ "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -33,12 +40,17 @@ "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -114,6 +126,198 @@ "recordedRepoMappingEntries": [] } }, + "@@rules_go~//go:extensions.bzl%go_sdk": { + "os:linux,arch:amd64": { + "bzlTransitiveDigest": "ngSfLTe8hQiidDeEb6YUqr77yNHrYln0spUfoRZ8O5I=", + "usagesDigest": "eBPCRLKeBr4iU55b1S2aLCYk6yMHyuLgphY9KhjbwDU=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "io_bazel_rules_nogo": { + "bzlFile": "@@rules_go~//go/private:nogo.bzl", + "ruleClassName": "go_register_nogo", + "attributes": { + "nogo": "@io_bazel_rules_go//:default_nogo", + "includes": [ + "'@@//:__subpackages__'" + ], + "excludes": [] + } + }, + "rules_go__download_0_windows_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "rules_go__download_0_linux_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_default_sdk": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "experiments": [], + "patches": [], + "patch_strip": 0, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1", + "strip_prefix": "go" + } + }, + "rules_go__download_0_darwin_arm64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_host_compatible_sdk_label": { + "bzlFile": "@@rules_go~//go/private:extensions.bzl", + "ruleClassName": "host_compatible_toolchain", + "attributes": { + "toolchain": "@go_default_sdk//:ROOT" + } + }, + "rules_go__download_0_darwin_amd64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + }, + "go_toolchains": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_multiple_toolchains", + "attributes": { + "prefixes": [ + "_0000_go_default_sdk_", + "_0001_rules_go__download_0_darwin_amd64_", + "_0002_rules_go__download_0_darwin_arm64_", + "_0003_rules_go__download_0_linux_arm64_", + "_0004_rules_go__download_0_windows_amd64_", + "_0005_rules_go__download_0_windows_arm64_" + ], + "geese": [ + "", + "darwin", + "darwin", + "linux", + "windows", + "windows" + ], + "goarchs": [ + "", + "amd64", + "arm64", + "arm64", + "amd64", + "arm64" + ], + "sdk_repos": [ + "go_default_sdk", + "rules_go__download_0_darwin_amd64", + "rules_go__download_0_darwin_arm64", + "rules_go__download_0_linux_arm64", + "rules_go__download_0_windows_amd64", + "rules_go__download_0_windows_arm64" + ], + "sdk_types": [ + "remote", + "remote", + "remote", + "remote", + "remote", + "remote" + ], + "sdk_versions": [ + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1", + "1.21.1" + ] + } + }, + "rules_go__download_0_windows_amd64": { + "bzlFile": "@@rules_go~//go/private:sdk.bzl", + "ruleClassName": "go_download_sdk_rule", + "attributes": { + "goos": "", + "goarch": "", + "sdks": {}, + "urls": [ + "https://dl.google.com/go/{}" + ], + "version": "1.21.1" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "bazel_features~", + "bazel_features_globals", + "bazel_features~~version_extension~bazel_features_globals" + ], + [ + "bazel_features~", + "bazel_features_version", + "bazel_features~~version_extension~bazel_features_version" + ], + [ + "rules_go~", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_go~", + "io_bazel_rules_go", + "rules_go~" + ], + [ + "rules_go~", + "io_bazel_rules_go_bazel_features", + "bazel_features~" + ] + ] + } + }, "@@toolchain_utils~//toolchain/export:defs.bzl%toolchain_export": { "general": { "bzlTransitiveDigest": "n2fd+/jiAv/nvvi2WUR+VQxI3aTZGNuhuX/NuFhw5fM=", -- GitLab From 7ae1e356b5c3fd20ea74f7c2a13e278f7c273f4e Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 20 Jun 2024 16:29:20 +0200 Subject: [PATCH 03/12] feat: upload manifests rule New rule to upload multiple files trough provided `CurlUploadManifestInfo` and `CurlUploadManifestsInfo`. --- curl/upload/manifests/BUILD.bazel | 16 ++++ curl/upload/manifests/defs.bzl | 5 ++ curl/upload/manifests/nt.tmpl.bat | 4 + curl/upload/manifests/posix.tmpl.sh | 131 ++++++++++++++++++++++++++++ curl/upload/manifests/rule.bzl | 112 ++++++++++++++++++++++++ 5 files changed, 268 insertions(+) create mode 100644 curl/upload/manifests/BUILD.bazel create mode 100644 curl/upload/manifests/defs.bzl create mode 100644 curl/upload/manifests/nt.tmpl.bat create mode 100644 curl/upload/manifests/posix.tmpl.sh create mode 100644 curl/upload/manifests/rule.bzl diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel new file mode 100644 index 0000000..ab707f7 --- /dev/null +++ b/curl/upload/manifests/BUILD.bazel @@ -0,0 +1,16 @@ +exports_files([ + "posix.tmpl.sh", + "nt.tmpl.bat", +]) + +alias( + name = "template", + actual = select( + { + "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", + "//conditions:default": ":posix.tmpl.sh", + }, + no_match_error = "No script template available for `curl_upload_manifests`", + ), + visibility = ["//visibility:public"], +) diff --git a/curl/upload/manifests/defs.bzl b/curl/upload/manifests/defs.bzl new file mode 100644 index 0000000..0625998 --- /dev/null +++ b/curl/upload/manifests/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _manifests = "manifests") + +visibility("public") + +curl_upload_manifests = _manifests diff --git a/curl/upload/manifests/nt.tmpl.bat b/curl/upload/manifests/nt.tmpl.bat new file mode 100644 index 0000000..d522214 --- /dev/null +++ b/curl/upload/manifests/nt.tmpl.bat @@ -0,0 +1,4 @@ +@echo off + +# TODO: implement Windows Batch for `curl_upload` +exit /b 121 diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/manifests/posix.tmpl.sh new file mode 100644 index 0000000..08c2d4e --- /dev/null +++ b/curl/upload/manifests/posix.tmpl.sh @@ -0,0 +1,131 @@ +#! /usr/bin/env sh + +# Strict shell +set -o errexit -o nounset + +# Bazel substitutions +CURL="{{curl}}" +UPLOAD_CSV="{{upload_csv}}" +RETRY="{{retry}}" +RETRY_DELAY="{{retry_delay}}" +TEMPLATE="{{template}}" +readonly CURL RETRY RETRY_DELAY UPLOAD_CSV + +# Runfiles +RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" +REPO_MAPPING="${RUNFILES_DIR}/_repo_mapping" +readonly RUNFILES_DIR REPO_MAPPING + +# Repository Mapping +if test -f "${REPO_MAPPING}"; then + while IFS=, read -r CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL; do + if test "${TARGET_LOCAL}" != "__main__"; then + printf >&2 'Error: Unexpected repository mapping: %s,%s,%s\n' "${CURRENT_CANONICAL}" "${TARGET_LOCAL}" "${TARGET_CANONICAL}" + exit 2 + fi + RUNFILES="${RUNFILES_DIR}/${TARGET_CANONICAL}" + break + done <"${REPO_MAPPING}" + unset CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL +else + RUNFILES="${0%/*}" +fi + +OVR_URL="" +OVR_TMPL="" +OVR_HOSTNAME="" +OVR_PORT="" +OVR_SCHEME="" +OVR_PATHNAME="" +OVR_FILE="" +OVR_BASENAME="" +OVR_EXTENSION="" +ARGS="" + +# Parse arguments +while test 0 -ne "${#}"; do + case "${1}" in + "--url") + shift + OVR_URL="${1?Must provide an argument for --url}" + ;; + "--hostname") + shift + OVR_HOSTNAME="${1?Must provide an argument for --hostname}" + ARGS="${ARGS} --hostname ${OVR_HOSTNAME}" + ;; + "--port") + shift + OVR_PORT="${1?Must provide an argument for --port}" + ARGS="${ARGS} -port ${OVR_PORT}" + ;; + "--scheme") + shift + OVR_SCHEME="${1?Must provide an argument for --scheme}" + ARGS="${ARGS} --scheme ${OVR_SCHEME}" + ;; + "--pathname") + shift + OVR_PATHNAME="${1?Must provide an argument for --pathname}" + ARGS="${ARGS} --pathname ${OVR_PATHNAME}" + ;; + "--file") + shift + OVR_FILE="${1?Must provide an argument for --file}" + ;; + "--basename") + shift + OVR_BASENAME="${1?Must provide an argument for --basename}" + ARGS="${ARGS} --basename ${OVR_BASENAME}" + ;; + "--extension") + shift + OVR_EXTENSION="${1?Must provide an argument for --extension}" + ARGS="${ARGS} --extension ${OVR_EXTENSION}" + ;; + "--template") + shift + OVR_TMPL="${1?Must provide an argument for --template}" + ;; + *) + printf >&2 'Error: unknown argument: %s\n' "${1}" + exit 2 + ;; + esac + shift +done +readonly OVR_URL OVR_HOSTNAME OVR_PORT OVR_SCHEME OVR_PATHNAME OVR_FILE OVR_BASENAME OVR_EXTENSION OVR_TMPL + +upload() { + + printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" + + # Do the upload + "${RUNFILES}/${CURL}" \ + --netrc \ + --location \ + --progress-bar \ + --retry "${RETRY}" \ + --retry-delay "${RETRY_DELAY}" \ + --upload-file "${RUNFILES}/${1}" \ + "${2}" +} + +while IFS=, read -r SRC TMPL URL; do + # TODO: `upload.csv` without `'` characters + SRC="${SRC#\'}" + URL="${URL%\'}" + set -f + IFS=' ' + set -- "${ARGS}" + DST=$( + "${RUNFILES}/${TEMPLATE}" \ + --url "${OVR_URL:-${URL}}" \ + --template "${OVR_TMPL:-${TMPL}}" \ + --file "${OVR_FILE:-${SRC}}" \ + "${@}" + ) + set +f + unset IFS + upload "${SRC}" "${DST}" +done <"${RUNFILES}/${UPLOAD_CSV}" diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl new file mode 100644 index 0000000..7b69626 --- /dev/null +++ b/curl/upload/manifests/rule.bzl @@ -0,0 +1,112 @@ +load("//curl/upload:ManifestInfo.bzl", "ManifestInfo") +load("//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") + +visibility("//curl/...") + +DOC = """Upload bunch of files to a URL endpoint with cURL. + +The `srcs` must provide `ManifestInfo` or `ManifestsInfo`. + +```py +file( + name = "upload_files", + srcs = [ + ":data", + ], + dst = "path/to/prefix" + url = "https://host.name.to.upload", +) +``` +""" + +ATTRS = { + "srcs": attr.label_list( + doc = "Files to be uploaded.", + mandatory = True, + providers = [ + [ManifestInfo], + [ManifestsInfo], + ], + ), + "dst": attr.string( + doc = "The destination directory to upload.", + default = "", + ), + "url": attr.string( + doc = "URL endpoint for files to upload.", + mandatory = True, + ), + "retry": attr.int( + doc = "The number of retry attempts for every file.", + default = 3, + ), + "retry_delay": attr.int( + doc = "The seconds to wait before attempting an upload retry.", + default = 1, + ), + "template": attr.label( + doc = "The template that is expanded into the upload binary.", + default = ":template", + allow_single_file = True, + ), + "_template": attr.label( + default = "//curl/template:template", + cfg = "exec", + allow_single_file = True, + executable = True, + ), +} + +def implementation(ctx): + curl = ctx.toolchains["//curl/toolchain/curl:type"] + + upload_csv = ctx.actions.declare_file("upload.csv") + manifests = depset( + direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], + transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], + ) + + href = ctx.attr.url.rstrip("/").replace(",", "%2C") + dst = ctx.attr.dst.rstrip("/") + + def _to_string(m): + # SRC,URL_TEMPL,URL + return "{},{},{}/{}".format(m.file.short_path, m.url, href, dst) + + args = ctx.actions.args() + args.add_all(manifests, map_each = _to_string, allow_closure = True) + ctx.actions.write(upload_csv, args, is_executable = False) + + executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) + + substitutions = ctx.actions.template_dict() + substitutions.add("{{curl}}", str(curl.executable.short_path)) + substitutions.add("{{retry}}", str(ctx.attr.retry)) + substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) + substitutions.add("{{upload_csv}}", str(upload_csv.short_path)) + substitutions.add("{{template}}", str(ctx.executable._template.short_path)) + + ctx.actions.expand_template( + template = ctx.file.template, + output = executable, + computed_substitutions = substitutions, + is_executable = True, + ) + + files = depset([executable]) + runfiles = ctx.runfiles([curl.executable, upload_csv, ctx.executable._template] + ctx.files.srcs + [m.file for m in manifests.to_list()]) + runfiles = runfiles.merge(curl.default.default_runfiles) + + return DefaultInfo( + executable = executable, + files = files, + runfiles = runfiles, + ) + +manifests = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, + toolchains = ["//curl/toolchain/curl:type"], + executable = True, +) -- GitLab From d6a430d96096190a7bcb2f9b4799c78448badb2f Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Thu, 20 Jun 2024 16:31:49 +0200 Subject: [PATCH 04/12] chore: test manifests upload --- e2e/MODULE.bazel | 1 + e2e/MODULE.bazel.lock | 32 +++++++++++++- e2e/mock/manifest/BUILD.bazel | 0 e2e/mock/manifest/rule.bzl | 33 +++++++++++++++ e2e/mock/manifests/BUILD.bazel | 0 e2e/mock/manifests/rule.bzl | 57 +++++++++++++++++++++++++ e2e/upload/file/BUILD.bazel | 9 ++-- e2e/upload/manifests/BUILD.bazel | 72 ++++++++++++++++++++++++++++++++ e2e/upload/manifests/fixture.txt | 60 ++++++++++++++++++++++++++ 9 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 e2e/mock/manifest/BUILD.bazel create mode 100644 e2e/mock/manifest/rule.bzl create mode 100644 e2e/mock/manifests/BUILD.bazel create mode 100644 e2e/mock/manifests/rule.bzl create mode 100644 e2e/upload/manifests/BUILD.bazel create mode 100644 e2e/upload/manifests/fixture.txt diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index 03a276e..cec2f49 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -7,6 +7,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") +bazel_dep(name = "rules_diff", version = "1.0.0-beta.3") bazel_dep(name = "rules_curl") local_path_override( module_name = "rules_curl", diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index 50ef842..e032719 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -47,6 +47,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/MODULE.bazel": "4bcae1c5e3c7fa1169f9940f548f7b8b3316944b4367771b168f925b7a9ee74e", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/source.json": "b5db3fcd469061f2051188da97345162f294fc59e7fdf477beb306dbe950566a", "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", @@ -321,7 +323,7 @@ "@@toolchain_utils~//toolchain/export:defs.bzl%toolchain_export": { "general": { "bzlTransitiveDigest": "n2fd+/jiAv/nvvi2WUR+VQxI3aTZGNuhuX/NuFhw5fM=", - "usagesDigest": "A2BbbNuBP0QFNqjCVowoaEOrolUDkTAis6lyamxEXLE=", + "usagesDigest": "OT9pdXtWdZUfYUSpReUZ6+fci56uJ5AVYR22QqnoBYI=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -578,6 +580,13 @@ "target": "@@ape~~_repo_rules~seq//:seq" } }, + "diff": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-diff//:ape-diff" + } + }, "ape-printenv": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", @@ -753,6 +762,13 @@ "target": "@@ape~~_repo_rules~qjs//:qjs" } }, + "cmp": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-cmp//:ape-cmp" + } + }, "ape-df": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", @@ -942,6 +958,20 @@ "target": "@@ape~~_repo_rules~nohup//:nohup" } }, + "sdiff": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-sdiff//:ape-sdiff" + } + }, + "diff3": { + "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", + "ruleClassName": "symlink", + "attributes": { + "target": "@@toolchain_utils~~toolchain_export~ape-diff3//:ape-diff3" + } + }, "ape-basename": { "bzlFile": "@@toolchain_utils~//toolchain/export/symlink:repository.bzl", "ruleClassName": "symlink", diff --git a/e2e/mock/manifest/BUILD.bazel b/e2e/mock/manifest/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl new file mode 100644 index 0000000..d64d819 --- /dev/null +++ b/e2e/mock/manifest/rule.bzl @@ -0,0 +1,33 @@ +load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo") + +DOC = """Mock rule with output `ManifestInfo` provider. + +```py +mock_manifest( + name = "mock_file", +) +``` +""" + +def _impl(ctx): + out = ctx.actions.declare_file("{}.out".format(ctx.attr.name)) + + ctx.actions.write(out, "hello test", is_executable = False) + + return [ + DefaultInfo( + files = depset([out]), + ), + ManifestInfo( + file = out, + url = "{{Href}}/{{.File.Path}}", + ), + ] + +mock_manifest = rule( + doc = DOC, + implementation = _impl, + provides = [ + ManifestInfo, + ], +) diff --git a/e2e/mock/manifests/BUILD.bazel b/e2e/mock/manifests/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl new file mode 100644 index 0000000..260dacf --- /dev/null +++ b/e2e/mock/manifests/rule.bzl @@ -0,0 +1,57 @@ +load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info") +load("@rules_curl//curl/upload:ManifestsInfo.bzl", "ManifestsInfo") + +DOC = """Mock rule with output `ManifestsInfo` provider. + +```py +mock_manifests( + name = "mock_files", + outs = [ + "file_1.out", + "file_2.out", + ], + deps = [ + "my_dep", + ], +) +``` +""" + +ATTRS = { + "outs": attr.string_list( + default = [], + mandatory = True, + ), + "deps": attr.label_list(allow_files = True), +} + +def _impl(ctx): + outs = [ctx.actions.declare_file(o) for o in ctx.attr.outs] + + for f in outs: + ctx.actions.write(f, "hello test", is_executable = False) + + files = depset(outs) + + manifests = [manifest_info(file = o, url = "{{Href}}/{{.File.Path}}") for o in outs] + manifests.extend([dep[ManifestInfo] for dep in ctx.attr.deps if ManifestInfo in dep]) + + transitive = [dep[ManifestsInfo].manifests for dep in ctx.attr.deps if ManifestsInfo in dep] + + return [ + DefaultInfo( + files = files, + ), + ManifestsInfo( + manifests = depset(direct = manifests, transitive = transitive), + ), + ] + +mock_manifests = rule( + doc = DOC, + attrs = ATTRS, + implementation = _impl, + provides = [ + ManifestsInfo, + ], +) diff --git a/e2e/upload/file/BUILD.bazel b/e2e/upload/file/BUILD.bazel index 6259557..6826b08 100644 --- a/e2e/upload/file/BUILD.bazel +++ b/e2e/upload/file/BUILD.bazel @@ -1,5 +1,5 @@ -load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") +load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") curl_upload_file( name = "upload", @@ -19,8 +19,9 @@ genrule( tools = [":upload"], ) -diff_test( +diff_file_test( name = "test", - file1 = ":fixture.txt", - file2 = ":execute", + size = "small", + a = ":fixture.txt", + b = ":execute", ) diff --git a/e2e/upload/manifests/BUILD.bazel b/e2e/upload/manifests/BUILD.bazel new file mode 100644 index 0000000..63ffa8a --- /dev/null +++ b/e2e/upload/manifests/BUILD.bazel @@ -0,0 +1,72 @@ +load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") +load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") +load("//mock/manifest:rule.bzl", "mock_manifest") +load("//mock/manifests:rule.bzl", "mock_manifests") + +genrule( + name = "data", + testonly = True, + outs = [ + "data.out", + ], + cmd = "echo 'Hello data\n' > $@", +) + +mock_manifest( + name = "fixture_1", +) + +mock_manifest( + name = "fixture_2", +) + +mock_manifests( + name = "file_pack_1", + testonly = True, + outs = [ + "file_1.out", + "file_2.out", + ], +) + +mock_manifests( + name = "file_pack_2", + testonly = True, + outs = [ + "file_A.txt", + "file_B.txt", + ], + deps = [ + ":data", + ":file_pack_1", + ":fixture_2", + ], +) + +curl_upload_manifests( + name = "upload_files", + testonly = True, + srcs = [ + ":file_pack_2", + ":fixture_1", + ], + dst = "final/destination/directory/", + url = "https://test.case", +) + +genrule( + name = "execute", + testonly = True, + outs = [ + "upload_files.out", + ], + cmd = "./$(location :upload_files) > $@", + tools = [":upload_files"], +) + +diff_file_test( + name = "test", + size = "small", + a = ":fixture.txt", + b = ":execute", +) diff --git a/e2e/upload/manifests/fixture.txt b/e2e/upload/manifests/fixture.txt new file mode 100644 index 0000000..eb2ff42 --- /dev/null +++ b/e2e/upload/manifests/fixture.txt @@ -0,0 +1,60 @@ +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_1.out +https://test.case/final/destination/directory/upload/manifests/file_1.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_2.out +https://test.case/final/destination/directory/upload/manifests/file_2.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_A.txt +https://test.case/final/destination/directory/upload/manifests/file_A.txt +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +file_B.txt +https://test.case/final/destination/directory/upload/manifests/file_B.txt +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +fixture_2.out +https://test.case/final/destination/directory/upload/manifests/fixture_2.out +--netrc +--location +--progress-bar +--retry +3 +--retry-delay +1 +--upload-file +fixture_1.out +https://test.case/final/destination/directory/upload/manifests/fixture_1.out -- GitLab From 7f47709578a05b1eb8ddfbf49892ba959270db38 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Thu, 20 Jun 2024 16:43:47 +0100 Subject: [PATCH 05/12] refactor: getters in `template.go` Allows templating with `{{.Url.Origin}}` for example. --- curl/template/template.go | 139 ++++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 34 deletions(-) diff --git a/curl/template/template.go b/curl/template/template.go index c66628e..87e5753 100644 --- a/curl/template/template.go +++ b/curl/template/template.go @@ -7,45 +7,113 @@ import ( "net/url" "os" "path/filepath" + "strconv" + "strings" ) type StringInfo string +func (s StringInfo) String() string { + return string(s) +} + +func (s *StringInfo) Set(value string) error { + *s = StringInfo(value) + return nil +} + type FileInfo struct { - Path StringInfo - Basename StringInfo + Dirname StringInfo + Stem StringInfo Extension StringInfo } +func (f FileInfo) Basename() string { + return f.Stem.String() + "." + f.Extension.String() +} + +func (f FileInfo) Path() string { + return f.Dirname.String() + "/" + f.Basename() +} + +func (f FileInfo) String() string { + return f.Path() +} + +func (f *FileInfo) Set(value string) error { + clean := filepath.Clean(value) + dirname := filepath.Dir(clean) + f.Dirname = StringInfo(dirname) + + extension := filepath.Ext(value) + nodot := strings.TrimPrefix(extension, ".") + f.Extension = StringInfo(nodot) + + base := filepath.Base(value) + stem := strings.TrimSuffix(base, extension) + f.Stem = StringInfo(stem) + return nil +} + type UrlInfo struct { - Hostname StringInfo - Port StringInfo + Username *StringInfo + Password *StringInfo + Host StringInfo Pathname StringInfo Protocol StringInfo } -func (u *UrlInfo) Href() string { - return u.Protocol.String() + "//" + u.Hostname.String() + u.Port.String() + u.Pathname.String() +func (u UrlInfo) split() (string, *int) { + host := u.Host.String() + index := strings.LastIndexByte(host, ':') + + if index == -1 { + return host, nil + } + + hostname := host[:index] + port, err := strconv.Atoi(host[index+1:]) + if err != nil { + log.Fatal(err) + } + return hostname, &port +} + +func (u UrlInfo) Hostname() string { + hostname, _ := u.split() + return hostname } -type ManifestInfo struct { - Url UrlInfo - File FileInfo +func (u UrlInfo) Port() *int { + _, port := u.split() + return port } -func (f *FileInfo) String() string { - return string(f.Path) +func (u UrlInfo) Origin() string { + return u.Protocol.String() + "//" + u.Host.String() } -func (f *FileInfo) Set(value string) error { - cleaned := filepath.Clean(value) - f.Path = StringInfo(filepath.ToSlash(cleaned)) - f.Basename = StringInfo(filepath.ToSlash(filepath.Base(cleaned))) - f.Extension = StringInfo(filepath.ToSlash(filepath.Ext(cleaned))) - return nil +func (u UrlInfo) Auth() (auth string) { + if u.Username != nil { + auth += string(*u.Username) + } + if u.Password != nil { + auth += ":" + string(*u.Password) + } + return +} + +func (u UrlInfo) Href() (href string) { + href += u.Protocol.String() + "//" + if auth := u.Auth(); auth != "" { + href += auth + "@" + } + href += u.Host.String() + href += u.Pathname.String() + return } -func (u *UrlInfo) String() string { +func (u UrlInfo) String() string { return u.Href() } @@ -54,39 +122,42 @@ func (u *UrlInfo) Set(value string) error { if err != nil { return err } - u.Hostname = StringInfo(parsed.Hostname()) - u.Port = StringInfo(parsed.Port()) + if username := parsed.User.Username(); username != "" { + info := StringInfo(username) + u.Username = &info + } + if password, set := parsed.User.Password(); set { + info := StringInfo(password) + u.Password = &info + } + u.Host = StringInfo(parsed.Host) u.Pathname = StringInfo(parsed.Path) u.Protocol = StringInfo(parsed.Scheme + ":") return err } -func (s *StringInfo) String() string { - return string(*s) -} - -func (s *StringInfo) Set(value string) error { - log.Fatal(value) - *s = StringInfo(value) - return nil +type ManifestInfo struct { + Url UrlInfo + File FileInfo } func main() { - + // TODO: can we make this global and use it to allow templating in `StringInfo.Set()`? manifest := ManifestInfo{ Url: UrlInfo{}, File: FileInfo{}, } + // TODO: is it possible for flags that modify `--origin` with a custom setter? flag.Var(&manifest.File, "file", "The file path to use for templating") - flag.Var(&manifest.File.Basename, "basename", "The basename of the source file") - flag.Var(&manifest.File.Extension, "extension", "The extension of the source file") + flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file") + flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file") + flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file") flag.Var(&manifest.Url, "url", "URL to use for templating") - flag.Var(&manifest.Url.Hostname, "hostname", "The domain name of the URL") - flag.Var(&manifest.Url.Port, "port", "The port number of the URL") + flag.Var(&manifest.Url.Host, "host", "The domain name of the URL") flag.Var(&manifest.Url.Pathname, "pathname", "A location in a hierachical structure") flag.Var(&manifest.Url.Protocol, "scheme", "The scheme for the URL") - t := flag.String("template", "{{Href}}/{{.File.Path}}", "The Go template to render") + t := flag.String("template", "{{.Url.Href}}/{{.File.Path}}", "The Go template to render") flag.Parse() -- GitLab From 9884f6020741990f4b1af3ee988ccbe5310eba05 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 21 Jun 2024 16:53:16 +0200 Subject: [PATCH 06/12] chore: update `rules_go` Update `rules_go` to latest `0.48.1`. --- MODULE.bazel | 2 +- MODULE.bazel.lock | 203 ++---------------------------------------- e2e/MODULE.bazel.lock | 203 ++---------------------------------------- 3 files changed, 17 insertions(+), 391 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 5614dd7..b2f1974 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,7 +8,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") bazel_dep(name = "ape", version = "1.0.0-beta.6") -bazel_dep(name = "rules_go", version = "0.46.0") +bazel_dep(name = "rules_go", version = "0.48.1") export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export") use_repo(export, "ape-curl") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 68eddf6..ff86585 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -13,6 +13,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", + "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", @@ -28,7 +30,8 @@ "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", - "https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.36.0/source.json": "0823f097b127e0201ae55d85647c94095edfe27db0431a7ae880dcab08dfaa04", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -50,7 +53,8 @@ "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", - "https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e", + "https://bcr.bazel.build/modules/rules_go/0.48.1/MODULE.bazel": "ad27296e268624d7d53043fe5ff88d5486e7a29596336f629b379b83c67e6d8b", + "https://bcr.bazel.build/modules/rules_go/0.48.1/source.json": "83321289aa500090871d8f761d991f0534946414640cce5c18d2df44cff8e082", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -63,7 +67,8 @@ "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/source.json": "de77e10ff0ab16acbf54e6b46eecd37a99c5b290468ea1aee6e95eb1affdaed7", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014", @@ -125,198 +130,6 @@ }, "recordedRepoMappingEntries": [] } - }, - "@@rules_go~//go:extensions.bzl%go_sdk": { - "os:linux,arch:amd64": { - "bzlTransitiveDigest": "ngSfLTe8hQiidDeEb6YUqr77yNHrYln0spUfoRZ8O5I=", - "usagesDigest": "eBPCRLKeBr4iU55b1S2aLCYk6yMHyuLgphY9KhjbwDU=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "io_bazel_rules_nogo": { - "bzlFile": "@@rules_go~//go/private:nogo.bzl", - "ruleClassName": "go_register_nogo", - "attributes": { - "nogo": "@io_bazel_rules_go//:default_nogo", - "includes": [ - "'@@//:__subpackages__'" - ], - "excludes": [] - } - }, - "rules_go__download_0_windows_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "rules_go__download_0_linux_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_default_sdk": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "experiments": [], - "patches": [], - "patch_strip": 0, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1", - "strip_prefix": "go" - } - }, - "rules_go__download_0_darwin_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_host_compatible_sdk_label": { - "bzlFile": "@@rules_go~//go/private:extensions.bzl", - "ruleClassName": "host_compatible_toolchain", - "attributes": { - "toolchain": "@go_default_sdk//:ROOT" - } - }, - "rules_go__download_0_darwin_amd64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_toolchains": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_multiple_toolchains", - "attributes": { - "prefixes": [ - "_0000_go_default_sdk_", - "_0001_rules_go__download_0_darwin_amd64_", - "_0002_rules_go__download_0_darwin_arm64_", - "_0003_rules_go__download_0_linux_arm64_", - "_0004_rules_go__download_0_windows_amd64_", - "_0005_rules_go__download_0_windows_arm64_" - ], - "geese": [ - "", - "darwin", - "darwin", - "linux", - "windows", - "windows" - ], - "goarchs": [ - "", - "amd64", - "arm64", - "arm64", - "amd64", - "arm64" - ], - "sdk_repos": [ - "go_default_sdk", - "rules_go__download_0_darwin_amd64", - "rules_go__download_0_darwin_arm64", - "rules_go__download_0_linux_arm64", - "rules_go__download_0_windows_amd64", - "rules_go__download_0_windows_arm64" - ], - "sdk_types": [ - "remote", - "remote", - "remote", - "remote", - "remote", - "remote" - ], - "sdk_versions": [ - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1" - ] - } - }, - "rules_go__download_0_windows_amd64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features~", - "bazel_features_globals", - "bazel_features~~version_extension~bazel_features_globals" - ], - [ - "bazel_features~", - "bazel_features_version", - "bazel_features~~version_extension~bazel_features_version" - ], - [ - "rules_go~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_go~", - "io_bazel_rules_go", - "rules_go~" - ], - [ - "rules_go~", - "io_bazel_rules_go_bazel_features", - "bazel_features~" - ] - ] - } } } } diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index e032719..f6b5460 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -13,6 +13,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", + "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", @@ -28,7 +30,8 @@ "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", - "https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.36.0/source.json": "0823f097b127e0201ae55d85647c94095edfe27db0431a7ae880dcab08dfaa04", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -52,7 +55,8 @@ "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", - "https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e", + "https://bcr.bazel.build/modules/rules_go/0.48.1/MODULE.bazel": "ad27296e268624d7d53043fe5ff88d5486e7a29596336f629b379b83c67e6d8b", + "https://bcr.bazel.build/modules/rules_go/0.48.1/source.json": "83321289aa500090871d8f761d991f0534946414640cce5c18d2df44cff8e082", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -65,7 +69,8 @@ "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/source.json": "de77e10ff0ab16acbf54e6b46eecd37a99c5b290468ea1aee6e95eb1affdaed7", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014", @@ -128,198 +133,6 @@ "recordedRepoMappingEntries": [] } }, - "@@rules_go~//go:extensions.bzl%go_sdk": { - "os:linux,arch:amd64": { - "bzlTransitiveDigest": "ngSfLTe8hQiidDeEb6YUqr77yNHrYln0spUfoRZ8O5I=", - "usagesDigest": "eBPCRLKeBr4iU55b1S2aLCYk6yMHyuLgphY9KhjbwDU=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "io_bazel_rules_nogo": { - "bzlFile": "@@rules_go~//go/private:nogo.bzl", - "ruleClassName": "go_register_nogo", - "attributes": { - "nogo": "@io_bazel_rules_go//:default_nogo", - "includes": [ - "'@@//:__subpackages__'" - ], - "excludes": [] - } - }, - "rules_go__download_0_windows_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "rules_go__download_0_linux_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_default_sdk": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "experiments": [], - "patches": [], - "patch_strip": 0, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1", - "strip_prefix": "go" - } - }, - "rules_go__download_0_darwin_arm64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_host_compatible_sdk_label": { - "bzlFile": "@@rules_go~//go/private:extensions.bzl", - "ruleClassName": "host_compatible_toolchain", - "attributes": { - "toolchain": "@go_default_sdk//:ROOT" - } - }, - "rules_go__download_0_darwin_amd64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - }, - "go_toolchains": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_multiple_toolchains", - "attributes": { - "prefixes": [ - "_0000_go_default_sdk_", - "_0001_rules_go__download_0_darwin_amd64_", - "_0002_rules_go__download_0_darwin_arm64_", - "_0003_rules_go__download_0_linux_arm64_", - "_0004_rules_go__download_0_windows_amd64_", - "_0005_rules_go__download_0_windows_arm64_" - ], - "geese": [ - "", - "darwin", - "darwin", - "linux", - "windows", - "windows" - ], - "goarchs": [ - "", - "amd64", - "arm64", - "arm64", - "amd64", - "arm64" - ], - "sdk_repos": [ - "go_default_sdk", - "rules_go__download_0_darwin_amd64", - "rules_go__download_0_darwin_arm64", - "rules_go__download_0_linux_arm64", - "rules_go__download_0_windows_amd64", - "rules_go__download_0_windows_arm64" - ], - "sdk_types": [ - "remote", - "remote", - "remote", - "remote", - "remote", - "remote" - ], - "sdk_versions": [ - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1", - "1.21.1" - ] - } - }, - "rules_go__download_0_windows_amd64": { - "bzlFile": "@@rules_go~//go/private:sdk.bzl", - "ruleClassName": "go_download_sdk_rule", - "attributes": { - "goos": "", - "goarch": "", - "sdks": {}, - "urls": [ - "https://dl.google.com/go/{}" - ], - "version": "1.21.1" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features~", - "bazel_features_globals", - "bazel_features~~version_extension~bazel_features_globals" - ], - [ - "bazel_features~", - "bazel_features_version", - "bazel_features~~version_extension~bazel_features_version" - ], - [ - "rules_go~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_go~", - "io_bazel_rules_go", - "rules_go~" - ], - [ - "rules_go~", - "io_bazel_rules_go_bazel_features", - "bazel_features~" - ] - ] - } - }, "@@toolchain_utils~//toolchain/export:defs.bzl%toolchain_export": { "general": { "bzlTransitiveDigest": "n2fd+/jiAv/nvvi2WUR+VQxI3aTZGNuhuX/NuFhw5fM=", -- GitLab From dd841c8c9654b817105a9f8f8c56619ffe41f944 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Fri, 21 Jun 2024 08:50:28 +0200 Subject: [PATCH 07/12] refactor: upload manifests rule Change does the refactor of: - rename ManifestInfo.Url -> ManifestInfo.URL - pass arguments directly to `template` go binary - avoid configs with private rule attribute - providing the `curl_upload_manifests` export --- curl/template/template.go | 95 +++++++++++++++++++++-------- curl/upload/ManifestInfo.bzl | 21 ++++--- curl/upload/manifests/BUILD.bazel | 3 +- curl/upload/manifests/posix.tmpl.sh | 84 +++---------------------- curl/upload/manifests/rule.bzl | 28 ++++----- e2e/mock/manifest/rule.bzl | 2 +- e2e/mock/manifests/rule.bzl | 2 +- e2e/upload/manifests/BUILD.bazel | 1 - e2e/upload/manifests/fixture.txt | 12 ++-- 9 files changed, 110 insertions(+), 138 deletions(-) diff --git a/curl/template/template.go b/curl/template/template.go index 87e5753..fc2a054 100644 --- a/curl/template/template.go +++ b/curl/template/template.go @@ -1,16 +1,22 @@ package main import ( + "bytes" "flag" + "fmt" "html/template" "log" "net/url" - "os" "path/filepath" "strconv" "strings" ) +var manifest ManifestInfo = ManifestInfo{ + URL: URLInfo{}, + File: FileInfo{}, +} + type StringInfo string func (s StringInfo) String() string { @@ -18,7 +24,19 @@ func (s StringInfo) String() string { } func (s *StringInfo) Set(value string) error { - *s = StringInfo(value) + buf := new(bytes.Buffer) + + tmpl, err := template.New("templ").Parse(value) + if err != nil { + log.Fatal(err) + } + + err = tmpl.Execute(buf, manifest) + if err != nil { + log.Fatal(err) + } + + *s = StringInfo(buf.String()) return nil } @@ -55,7 +73,7 @@ func (f *FileInfo) Set(value string) error { return nil } -type UrlInfo struct { +type URLInfo struct { Username *StringInfo Password *StringInfo Host StringInfo @@ -63,7 +81,7 @@ type UrlInfo struct { Protocol StringInfo } -func (u UrlInfo) split() (string, *int) { +func (u URLInfo) split() (string, *int) { host := u.Host.String() index := strings.LastIndexByte(host, ':') @@ -79,21 +97,21 @@ func (u UrlInfo) split() (string, *int) { return hostname, &port } -func (u UrlInfo) Hostname() string { +func (u URLInfo) Hostname() string { hostname, _ := u.split() return hostname } -func (u UrlInfo) Port() *int { +func (u URLInfo) Port() *int { _, port := u.split() return port } -func (u UrlInfo) Origin() string { +func (u URLInfo) Origin() string { return u.Protocol.String() + "//" + u.Host.String() } -func (u UrlInfo) Auth() (auth string) { +func (u URLInfo) Auth() (auth string) { if u.Username != nil { auth += string(*u.Username) } @@ -103,7 +121,7 @@ func (u UrlInfo) Auth() (auth string) { return } -func (u UrlInfo) Href() (href string) { +func (u URLInfo) Href() (href string) { href += u.Protocol.String() + "//" if auth := u.Auth(); auth != "" { href += auth + "@" @@ -113,11 +131,11 @@ func (u UrlInfo) Href() (href string) { return } -func (u UrlInfo) String() string { +func (u URLInfo) String() string { return u.Href() } -func (u *UrlInfo) Set(value string) error { +func (u *URLInfo) Set(value string) error { parsed, err := url.Parse(value) if err != nil { return err @@ -137,39 +155,62 @@ func (u *UrlInfo) Set(value string) error { } type ManifestInfo struct { - Url UrlInfo + URL URLInfo File FileInfo } func main() { - // TODO: can we make this global and use it to allow templating in `StringInfo.Set()`? - manifest := ManifestInfo{ - Url: UrlInfo{}, - File: FileInfo{}, - } - - // TODO: is it possible for flags that modify `--origin` with a custom setter? flag.Var(&manifest.File, "file", "The file path to use for templating") flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file") flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file") flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file") - flag.Var(&manifest.Url, "url", "URL to use for templating") - flag.Var(&manifest.Url.Host, "host", "The domain name of the URL") - flag.Var(&manifest.Url.Pathname, "pathname", "A location in a hierachical structure") - flag.Var(&manifest.Url.Protocol, "scheme", "The scheme for the URL") - t := flag.String("template", "{{.Url.Href}}/{{.File.Path}}", "The Go template to render") + flag.Var(&manifest.URL, "url", "URL to use for templating") + flag.Var(&manifest.URL.Host, "host", "The domain name of the URL") + flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL") + flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error { + buf := new(bytes.Buffer) + if !strings.HasPrefix(s, "/") { + buf.WriteString("/") + } + buf.WriteString(s) + + manifest.URL.Pathname.Set(buf.String()) + + return nil + }) + flag.Func("origin", "The origin of the represented URL.", func(s string) error { + index := strings.Index(s, "//") + + if index == -1 { + manifest.URL.Protocol = StringInfo(s) + return nil + } + + manifest.URL.Host = StringInfo(s[index+2:]) + manifest.URL.Protocol = StringInfo(s[:index]) + + return nil + }) + t := flag.String("template", "{{.URL.Href}}/{{.File.Path}}", "The Go template to render") flag.Parse() - var funcs = template.FuncMap{"Href": manifest.Url.Href} + tmpl, err := template.New("url").Parse(*t) + if err != nil { + log.Fatal(err) + } - tmpl, err := template.New("url").Funcs(funcs).Parse(*t) + buf := new(bytes.Buffer) + + err = tmpl.Execute(buf, manifest) if err != nil { log.Fatal(err) } - err = tmpl.Execute(os.Stdout, manifest) + _, err = url.Parse(buf.String()) if err != nil { log.Fatal(err) } + + fmt.Printf("%s", buf.String()) } diff --git a/curl/upload/ManifestInfo.bzl b/curl/upload/ManifestInfo.bzl index 175f565..60d5d60 100644 --- a/curl/upload/ManifestInfo.bzl +++ b/curl/upload/ManifestInfo.bzl @@ -11,14 +11,19 @@ def init(file, url): Args: file: The file to upload. url: A Starlark string template for the URL, which can include: - - {{Href}}: the provided URL - - {{.Url.Hostname}}: the domain name of the URL - - {{.Url.Port}}: the port number of the URL - - {{.Url.Protocol}}: the protocol scheme of the URL - - {{.Url.Pathname}}: a location in a hierarchical structure - - {{.File.Path}}: the full path of the source file - - {{.File.Basename}}: the basename of the source file - - {{.File.Extension}}: the extension of the source file + - {{.URL.Href}}: the provided URL + - {{.URL.Auth}}: the user:password auth + - {{.URL.Origin}}: the origin of the represented URL + - {{.URL.Port}}: the port number of the URL + - {{.URL.Host}}: the hostname followed by the port if not empty + - {{.URL.Hostname}}: the domain name of the URL + - {{.URL.Protocol}}: the protocol scheme of the URL + - {{.URL.Pathname}}: a location in a hierarchical structure + - {{.File.Path}}: the full path of the file + - {{.File.Basename}}: the basename of the file + - {{.File.Dirname}}: the directory of the file + - {{.File.Stem}}: the file name without extension + - {{.File.Extension}}: the extension of the file Returns: A mapping of keywords for the `curl_upload_manifest_info` raw constructor. diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel index ab707f7..ac7a290 100644 --- a/curl/upload/manifests/BUILD.bazel +++ b/curl/upload/manifests/BUILD.bazel @@ -4,7 +4,7 @@ exports_files([ ]) alias( - name = "template", + name = "script", actual = select( { "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", @@ -12,5 +12,4 @@ alias( }, no_match_error = "No script template available for `curl_upload_manifests`", ), - visibility = ["//visibility:public"], ) diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/manifests/posix.tmpl.sh index 08c2d4e..6d48b81 100644 --- a/curl/upload/manifests/posix.tmpl.sh +++ b/curl/upload/manifests/posix.tmpl.sh @@ -5,11 +5,11 @@ set -o errexit -o nounset # Bazel substitutions CURL="{{curl}}" -UPLOAD_CSV="{{upload_csv}}" +CSV="{{csv}}" RETRY="{{retry}}" RETRY_DELAY="{{retry_delay}}" TEMPLATE="{{template}}" -readonly CURL RETRY RETRY_DELAY UPLOAD_CSV +readonly CURL RETRY RETRY_DELAY CSV # Runfiles RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" @@ -31,71 +31,6 @@ else RUNFILES="${0%/*}" fi -OVR_URL="" -OVR_TMPL="" -OVR_HOSTNAME="" -OVR_PORT="" -OVR_SCHEME="" -OVR_PATHNAME="" -OVR_FILE="" -OVR_BASENAME="" -OVR_EXTENSION="" -ARGS="" - -# Parse arguments -while test 0 -ne "${#}"; do - case "${1}" in - "--url") - shift - OVR_URL="${1?Must provide an argument for --url}" - ;; - "--hostname") - shift - OVR_HOSTNAME="${1?Must provide an argument for --hostname}" - ARGS="${ARGS} --hostname ${OVR_HOSTNAME}" - ;; - "--port") - shift - OVR_PORT="${1?Must provide an argument for --port}" - ARGS="${ARGS} -port ${OVR_PORT}" - ;; - "--scheme") - shift - OVR_SCHEME="${1?Must provide an argument for --scheme}" - ARGS="${ARGS} --scheme ${OVR_SCHEME}" - ;; - "--pathname") - shift - OVR_PATHNAME="${1?Must provide an argument for --pathname}" - ARGS="${ARGS} --pathname ${OVR_PATHNAME}" - ;; - "--file") - shift - OVR_FILE="${1?Must provide an argument for --file}" - ;; - "--basename") - shift - OVR_BASENAME="${1?Must provide an argument for --basename}" - ARGS="${ARGS} --basename ${OVR_BASENAME}" - ;; - "--extension") - shift - OVR_EXTENSION="${1?Must provide an argument for --extension}" - ARGS="${ARGS} --extension ${OVR_EXTENSION}" - ;; - "--template") - shift - OVR_TMPL="${1?Must provide an argument for --template}" - ;; - *) - printf >&2 'Error: unknown argument: %s\n' "${1}" - exit 2 - ;; - esac - shift -done -readonly OVR_URL OVR_HOSTNAME OVR_PORT OVR_SCHEME OVR_PATHNAME OVR_FILE OVR_BASENAME OVR_EXTENSION OVR_TMPL - upload() { printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" @@ -115,17 +50,12 @@ while IFS=, read -r SRC TMPL URL; do # TODO: `upload.csv` without `'` characters SRC="${SRC#\'}" URL="${URL%\'}" - set -f - IFS=' ' - set -- "${ARGS}" DST=$( "${RUNFILES}/${TEMPLATE}" \ - --url "${OVR_URL:-${URL}}" \ - --template "${OVR_TMPL:-${TMPL}}" \ - --file "${OVR_FILE:-${SRC}}" \ - "${@}" + --url "${URL}" \ + --template "${TMPL}" \ + --file "${SRC}" \ + ${@} ) - set +f - unset IFS upload "${SRC}" "${DST}" -done <"${RUNFILES}/${UPLOAD_CSV}" +done <"${RUNFILES}/${CSV}" diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 7b69626..308c325 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -13,7 +13,6 @@ file( srcs = [ ":data", ], - dst = "path/to/prefix" url = "https://host.name.to.upload", ) ``` @@ -27,10 +26,7 @@ ATTRS = { [ManifestInfo], [ManifestsInfo], ], - ), - "dst": attr.string( - doc = "The destination directory to upload.", - default = "", + allow_files = False, ), "url": attr.string( doc = "URL endpoint for files to upload.", @@ -44,9 +40,9 @@ ATTRS = { doc = "The seconds to wait before attempting an upload retry.", default = 1, ), - "template": attr.label( + "_script": attr.label( doc = "The template that is expanded into the upload binary.", - default = ":template", + default = ":script", allow_single_file = True, ), "_template": attr.label( @@ -60,22 +56,21 @@ ATTRS = { def implementation(ctx): curl = ctx.toolchains["//curl/toolchain/curl:type"] - upload_csv = ctx.actions.declare_file("upload.csv") + csv = ctx.actions.declare_file("upload.csv") manifests = depset( direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src], transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src], ) href = ctx.attr.url.rstrip("/").replace(",", "%2C") - dst = ctx.attr.dst.rstrip("/") def _to_string(m): # SRC,URL_TEMPL,URL - return "{},{},{}/{}".format(m.file.short_path, m.url, href, dst) + return "{},{},{}".format(m.file.short_path, m.url, href) args = ctx.actions.args() args.add_all(manifests, map_each = _to_string, allow_closure = True) - ctx.actions.write(upload_csv, args, is_executable = False) + ctx.actions.write(csv, args, is_executable = False) executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) @@ -83,19 +78,20 @@ def implementation(ctx): substitutions.add("{{curl}}", str(curl.executable.short_path)) substitutions.add("{{retry}}", str(ctx.attr.retry)) substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay)) - substitutions.add("{{upload_csv}}", str(upload_csv.short_path)) + substitutions.add("{{csv}}", str(csv.short_path)) substitutions.add("{{template}}", str(ctx.executable._template.short_path)) ctx.actions.expand_template( - template = ctx.file.template, + template = ctx.file._script, output = executable, computed_substitutions = substitutions, is_executable = True, ) files = depset([executable]) - runfiles = ctx.runfiles([curl.executable, upload_csv, ctx.executable._template] + ctx.files.srcs + [m.file for m in manifests.to_list()]) + runfiles = ctx.runfiles([curl.executable, csv, ctx.executable._template] + [m.file for m in manifests.to_list()]) runfiles = runfiles.merge(curl.default.default_runfiles) + runfiles = runfiles.merge(ctx.attr._template.default_runfiles) return DefaultInfo( executable = executable, @@ -103,10 +99,12 @@ def implementation(ctx): runfiles = runfiles, ) -manifests = rule( +curl_upload_manifests = rule( doc = DOC, attrs = ATTRS, implementation = implementation, toolchains = ["//curl/toolchain/curl:type"], executable = True, ) + +manifests = curl_upload_manifests diff --git a/e2e/mock/manifest/rule.bzl b/e2e/mock/manifest/rule.bzl index d64d819..6d3bbe2 100644 --- a/e2e/mock/manifest/rule.bzl +++ b/e2e/mock/manifest/rule.bzl @@ -20,7 +20,7 @@ def _impl(ctx): ), ManifestInfo( file = out, - url = "{{Href}}/{{.File.Path}}", + url = "{{.URL.Href}}/{{.File.Path}}", ), ] diff --git a/e2e/mock/manifests/rule.bzl b/e2e/mock/manifests/rule.bzl index 260dacf..8f3caea 100644 --- a/e2e/mock/manifests/rule.bzl +++ b/e2e/mock/manifests/rule.bzl @@ -33,7 +33,7 @@ def _impl(ctx): files = depset(outs) - manifests = [manifest_info(file = o, url = "{{Href}}/{{.File.Path}}") for o in outs] + manifests = [manifest_info(file = o, url = "{{.URL.Href}}/{{.File.Path}}") for o in outs] manifests.extend([dep[ManifestInfo] for dep in ctx.attr.deps if ManifestInfo in dep]) transitive = [dep[ManifestsInfo].manifests for dep in ctx.attr.deps if ManifestsInfo in dep] diff --git a/e2e/upload/manifests/BUILD.bazel b/e2e/upload/manifests/BUILD.bazel index 63ffa8a..b60e578 100644 --- a/e2e/upload/manifests/BUILD.bazel +++ b/e2e/upload/manifests/BUILD.bazel @@ -50,7 +50,6 @@ curl_upload_manifests( ":file_pack_2", ":fixture_1", ], - dst = "final/destination/directory/", url = "https://test.case", ) diff --git a/e2e/upload/manifests/fixture.txt b/e2e/upload/manifests/fixture.txt index eb2ff42..5dbf8f7 100644 --- a/e2e/upload/manifests/fixture.txt +++ b/e2e/upload/manifests/fixture.txt @@ -7,7 +7,7 @@ 1 --upload-file file_1.out -https://test.case/final/destination/directory/upload/manifests/file_1.out +https://test.case/upload/manifests/file_1.out --netrc --location --progress-bar @@ -17,7 +17,7 @@ https://test.case/final/destination/directory/upload/manifests/file_1.out 1 --upload-file file_2.out -https://test.case/final/destination/directory/upload/manifests/file_2.out +https://test.case/upload/manifests/file_2.out --netrc --location --progress-bar @@ -27,7 +27,7 @@ https://test.case/final/destination/directory/upload/manifests/file_2.out 1 --upload-file file_A.txt -https://test.case/final/destination/directory/upload/manifests/file_A.txt +https://test.case/upload/manifests/file_A.txt --netrc --location --progress-bar @@ -37,7 +37,7 @@ https://test.case/final/destination/directory/upload/manifests/file_A.txt 1 --upload-file file_B.txt -https://test.case/final/destination/directory/upload/manifests/file_B.txt +https://test.case/upload/manifests/file_B.txt --netrc --location --progress-bar @@ -47,7 +47,7 @@ https://test.case/final/destination/directory/upload/manifests/file_B.txt 1 --upload-file fixture_2.out -https://test.case/final/destination/directory/upload/manifests/fixture_2.out +https://test.case/upload/manifests/fixture_2.out --netrc --location --progress-bar @@ -57,4 +57,4 @@ https://test.case/final/destination/directory/upload/manifests/fixture_2.out 1 --upload-file fixture_1.out -https://test.case/final/destination/directory/upload/manifests/fixture_1.out +https://test.case/upload/manifests/fixture_1.out -- GitLab From 1f8540811837789004aac12f50ad1a99b5cbedfb Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Mon, 24 Jun 2024 16:03:55 +0200 Subject: [PATCH 08/12] chore: prepare `upload.csv` Writing `actions.Args` to file directly quotes every arg with. Use tool binary who builds correct ".csv" file. --- curl/upload/manifests/BUILD.bazel | 5 +++++ curl/upload/manifests/csv.sh | 11 +++++++++++ curl/upload/manifests/posix.tmpl.sh | 3 --- curl/upload/manifests/rule.bzl | 13 ++++++++++++- 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100755 curl/upload/manifests/csv.sh diff --git a/curl/upload/manifests/BUILD.bazel b/curl/upload/manifests/BUILD.bazel index ac7a290..1a5d7c7 100644 --- a/curl/upload/manifests/BUILD.bazel +++ b/curl/upload/manifests/BUILD.bazel @@ -13,3 +13,8 @@ alias( no_match_error = "No script template available for `curl_upload_manifests`", ), ) + +sh_binary( + name = "csv", + srcs = ["csv.sh"], +) diff --git a/curl/upload/manifests/csv.sh b/curl/upload/manifests/csv.sh new file mode 100755 index 0000000..81e6de3 --- /dev/null +++ b/curl/upload/manifests/csv.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env sh + +set -eu -o errexit -o nounset + +OUT="${1}" +shift +readonly OUT + +for ARG in "${@}"; do + printf >>"${OUT}" '%s\n' "${ARG}" +done diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/manifests/posix.tmpl.sh index 6d48b81..5294d50 100644 --- a/curl/upload/manifests/posix.tmpl.sh +++ b/curl/upload/manifests/posix.tmpl.sh @@ -47,9 +47,6 @@ upload() { } while IFS=, read -r SRC TMPL URL; do - # TODO: `upload.csv` without `'` characters - SRC="${SRC#\'}" - URL="${URL%\'}" DST=$( "${RUNFILES}/${TEMPLATE}" \ --url "${URL}" \ diff --git a/curl/upload/manifests/rule.bzl b/curl/upload/manifests/rule.bzl index 308c325..9a4ac1c 100644 --- a/curl/upload/manifests/rule.bzl +++ b/curl/upload/manifests/rule.bzl @@ -51,6 +51,12 @@ ATTRS = { allow_single_file = True, executable = True, ), + "_csv": attr.label( + doc = "CSV tool", + default = ":csv", + cfg = "exec", + executable = True, + ), } def implementation(ctx): @@ -70,7 +76,12 @@ def implementation(ctx): args = ctx.actions.args() args.add_all(manifests, map_each = _to_string, allow_closure = True) - ctx.actions.write(csv, args, is_executable = False) + + ctx.actions.run( + outputs = [csv], + arguments = [csv.path, args], + executable = ctx.executable._csv, + ) executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) -- GitLab From 8a427740d719f2d05171c6b64685c971b48aed7d Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Mon, 24 Jun 2024 16:11:17 +0200 Subject: [PATCH 09/12] fix: set `pure` for `template` --- curl/template/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/curl/template/BUILD.bazel b/curl/template/BUILD.bazel index fc9c00b..d22934d 100644 --- a/curl/template/BUILD.bazel +++ b/curl/template/BUILD.bazel @@ -3,5 +3,6 @@ load("@rules_go//go:def.bzl", "go_binary") go_binary( name = "template", srcs = ["template.go"], + pure = "on", visibility = ["//curl/upload:__subpackages__"], ) -- GitLab From 009e995d4cbcacae95933d6ab9c7379b90a99a51 Mon Sep 17 00:00:00 2001 From: Sebastian Birunt Date: Tue, 25 Jun 2024 10:55:02 +0200 Subject: [PATCH 10/12] chore: update docs --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 712d213..d88cecd 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,57 @@ Add the following to `MODULE.bazel`: ```py bazel_dep(module_name = "rules_curl", version = "0.0.0") ``` + +## Providers + +### ManifestInfo + +Encapsulates the file to upload and its URL template (see [ManifestInfo][manifest_info]) + +Members: + +- **file**: File +- **url**: string + +### ManifestsInfo + +The dependency set of `ManifestInfo`s + +Members: + +- **manifests**: depset of `ManifestInfo`s + +## Usage + +### curl_upload_file + +```py +load("@rules_curl//curl/upload/file:defs.bzl", "curl_upload_file") + +curl_upload_file( + name = "upload", + src = ":fixture.txt", + dst = "directory/fixture.txt", + url = "https://test.case", +) + +``` + +### curl_upload_manifests + +```py +load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests") + +curl_upload_manifests( + name = "upload", + srcs = [ + ":fixture_1", + ":fixture_2", + ], + url = "https://test.case", +) +``` + +> Note: `fixture_1` and `fixture_2` have to provide either `ManifestInfo` or `ManifestsInfo` + +[manifest_info]: curl/upload/ManifestInfo.bzl -- GitLab From d2b90bc4690f8a20cefe69c388bb0055795d11f0 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 25 Jun 2024 12:27:49 +0100 Subject: [PATCH 11/12] chore: refactor to `rlocation for runfiles --- curl/upload/manifests/posix.tmpl.sh | 62 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/curl/upload/manifests/posix.tmpl.sh b/curl/upload/manifests/posix.tmpl.sh index 5294d50..d11c9ea 100644 --- a/curl/upload/manifests/posix.tmpl.sh +++ b/curl/upload/manifests/posix.tmpl.sh @@ -3,56 +3,60 @@ # Strict shell set -o errexit -o nounset +# Runfiles location +rlocation() ( + readonly FILEPATH="${1}" + + if test -x "${FILEPATH}"; then + printf '%s' "${FILEPATH}" + return + fi + + readonly RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" + + if test -x "${RUNFILES_DIR}/${FILEPATH#../}"; then + printf '%s' "${RUNFILES_DIR}/${FILEPATH#../}" + return + fi + + if test -x "${RUNFILES_DIR}/"*"/${FILEPATH#../}"; then + printf '%s' "${RUNFILES_DIR}/"*"/${FILEPATH#../}" + return + fi + + exit 1 +) + # Bazel substitutions -CURL="{{curl}}" -CSV="{{csv}}" +CURL="$(rlocation "{{curl}}")" +CSV="$(rlocation "{{csv}}")" +TEMPLATE="$(rlocation "{{template}}")" RETRY="{{retry}}" RETRY_DELAY="{{retry_delay}}" -TEMPLATE="{{template}}" -readonly CURL RETRY RETRY_DELAY CSV - -# Runfiles -RUNFILES_DIR="${RUNFILES_DIR-${0}.runfiles}" -REPO_MAPPING="${RUNFILES_DIR}/_repo_mapping" -readonly RUNFILES_DIR REPO_MAPPING - -# Repository Mapping -if test -f "${REPO_MAPPING}"; then - while IFS=, read -r CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL; do - if test "${TARGET_LOCAL}" != "__main__"; then - printf >&2 'Error: Unexpected repository mapping: %s,%s,%s\n' "${CURRENT_CANONICAL}" "${TARGET_LOCAL}" "${TARGET_CANONICAL}" - exit 2 - fi - RUNFILES="${RUNFILES_DIR}/${TARGET_CANONICAL}" - break - done <"${REPO_MAPPING}" - unset CURRENT_CANONICAL TARGET_LOCAL TARGET_CANONICAL -else - RUNFILES="${0%/*}" -fi +readonly CURL CSV TEMPLATE RETRY RETRY_DELAY +# Uploads a file upload() { - printf >&2 "Uploading: %s to %s\n" "${1}" "${2}" # Do the upload - "${RUNFILES}/${CURL}" \ + "${CURL}" \ --netrc \ --location \ --progress-bar \ --retry "${RETRY}" \ --retry-delay "${RETRY_DELAY}" \ - --upload-file "${RUNFILES}/${1}" \ + --upload-file "$(rlocation "${1}")" \ "${2}" } while IFS=, read -r SRC TMPL URL; do DST=$( - "${RUNFILES}/${TEMPLATE}" \ + "${TEMPLATE}" \ --url "${URL}" \ --template "${TMPL}" \ --file "${SRC}" \ ${@} ) upload "${SRC}" "${DST}" -done <"${RUNFILES}/${CSV}" +done <"${CSV}" -- GitLab From ebe36a9ddf0e4fa3404179debe3b468de30a0bcf Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 25 Jun 2024 12:46:37 +0100 Subject: [PATCH 12/12] test: add hermetic C/C++ toolchain --- e2e/MODULE.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index cec2f49..611bcfb 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -8,6 +8,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.9") bazel_dep(name = "rules_diff", version = "1.0.0-beta.3") +bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0") bazel_dep(name = "rules_curl") local_path_override( module_name = "rules_curl", -- GitLab