Add https binary cache (#558)

* Add https binary cache

* Fix naming mistake

* Add support for http

Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

include/vcpkg/base/api_stable_format.h

@@ -15,16 +15,15 @@ namespace vcpkg
             (*(F*)(f))(s, sv);
         }
 
-        ExpectedS<std::string> api_stable_format_impl(StringView fmtstr,
+        ExpectedL<std::string> api_stable_format_impl(StringView fmtstr,
                                                       void (*cb)(void*, std::string&, StringView),
                                                       void* data);
-
     }
 
     // This function exists in order to provide an API-stable formatting function similar to `std::format()` that does
     // not depend on the feature set of fmt or the C++ standard library and thus can be contractual for user interfaces.
     template<class F>
-    ExpectedS<std::string> api_stable_format(StringView fmtstr, F&& handler)
+    ExpectedL<std::string> api_stable_format(StringView fmtstr, F&& handler)
     {
         return details::api_stable_format_impl(fmtstr, &details::api_stable_format_cb<F>, &handler);
     }

include/vcpkg/base/downloads.h

@@ -32,7 +32,8 @@ namespace vcpkg
 
     View<std::string> azure_blob_headers();
 
-    std::vector<int> download_files(Filesystem& fs, View<std::pair<std::string, Path>> url_pairs);
+    std::vector<int> download_files(Filesystem& fs,
+                                    View<std::tuple<std::string, View<std::string>, Path>> url_header_path_tuples);
     ExpectedS<int> put_file(const Filesystem&, StringView url, View<std::string> headers, const Path& file);
     std::vector<int> url_heads(View<std::string> urls, View<std::string> headers, View<std::string> secrets);
     std::string replace_secrets(std::string input, View<std::string> secrets);

include/vcpkg/binarycaching.h

@@ -69,6 +69,15 @@ namespace vcpkg
         virtual void precheck(View<Dependencies::InstallPlanAction> actions, View<CacheStatus*> cache_status) const = 0;
     };
 
+    struct UrlTemplate
+    {
+        std::string url_template;
+        std::vector<std::string> headers;
+
+        LocalizedString valid();
+        std::string instantiate_variables(const Dependencies::InstallPlanAction& action) const;
+    };
+
     struct BinaryConfigParserState
     {
         bool m_cleared = false;
@@ -78,8 +87,8 @@ namespace vcpkg
         std::vector<Path> archives_to_read;
         std::vector<Path> archives_to_write;
 
-        std::vector<std::string> url_templates_to_get;
-        std::vector<std::string> azblob_templates_to_put;
+        std::vector<UrlTemplate> url_templates_to_get;
+        std::vector<UrlTemplate> url_templates_to_put;
 
         std::vector<std::string> gcs_read_prefixes;
         std::vector<std::string> gcs_write_prefixes;

locales/messages.en.json

@@ -145,6 +145,7 @@
   "InvalidArgumentRequiresSourceArgument": "invalid argument: binary config '{binary_source}' requires at least a source argument",
   "InvalidArgumentRequiresTwoOrThreeArguments": "invalid argument: binary config '{binary_source}' requires 2 or 3 arguments",
   "InvalidArgumentRequiresValidToken": "invalid argument: binary config '{binary_source}' requires a SAS token without a preceeding '?' as the second argument",
+  "InvalidFormatString": "invalid format string: {actual}",
   "JsonErrorFailedToParse": "failed to parse {path}:",
   "JsonErrorFailedToRead": "failed to read {path}: {error_msg}",
   "LaunchingProgramFailed": "Launching {tool_name}:",
@@ -198,6 +199,7 @@
   "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)'",
   "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'",
   "UnknownTool": "vcpkg does not have a definition of this tool for this platform.",
+  "UnknownVariablesInTemplate": "invalid argument: url template '{value}' contains unknown variables: {list}",
   "UnsupportedSystemName": "Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.",
   "UnsupportedToolchain": "in triplet {triplet}: Unable to find a valid toolchain for requested target architecture {arch}.\nThe selected Visual Studio instance is at: {path}\nThe available toolchain combinations are: {list}",
   "UpdateBaselineAddBaselineNoManifest": "the --{option} switch was passed, but there is no manifest file to add a `builtin-baseline` field to.",

locales/messages.json

@@ -255,6 +255,8 @@
   "_InvalidArgumentRequiresTwoOrThreeArguments.comment": "An example of {binary_source} is azblob.",
   "InvalidArgumentRequiresValidToken": "invalid argument: binary config '{binary_source}' requires a SAS token without a preceeding '?' as the second argument",
   "_InvalidArgumentRequiresValidToken.comment": "An example of {binary_source} is azblob.",
+  "InvalidFormatString": "invalid format string: {actual}",
+  "_InvalidFormatString.comment": "{actual} is the provided format string",
   "JsonErrorFailedToParse": "failed to parse {path}:",
   "_JsonErrorFailedToParse.comment": "An example of {path} is /foo/bar.",
   "JsonErrorFailedToRead": "failed to read {path}: {error_msg}",
@@ -341,6 +343,8 @@
   "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip|pass)'",
   "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'",
   "UnknownTool": "vcpkg does not have a definition of this tool for this platform.",
+  "UnknownVariablesInTemplate": "invalid argument: url template '{value}' contains unknown variables: {list}",
+  "_UnknownVariablesInTemplate.comment": "{value} is the value provided by the user and {list} a list of unknown variables seperated by comma",
   "UnsupportedSystemName": "Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.",
   "_UnsupportedSystemName.comment": "An example of {system_name} is Darwin.",
   "UnsupportedToolchain": "in triplet {triplet}: Unable to find a valid toolchain for requested target architecture {arch}.\nThe selected Visual Studio instance is at: {path}\nThe available toolchain combinations are: {list}",

src/vcpkg/base/downloads.cpp

@@ -353,7 +353,9 @@ namespace vcpkg
         return input;
     }
 
-    static void download_files_inner(Filesystem&, View<std::pair<std::string, Path>> url_pairs, std::vector<int>* out)
+    static void download_files_inner(Filesystem&,
+                                     View<std::tuple<std::string, View<std::string>, Path>> url_header_path_tuples,
+                                     std::vector<int>* out)
     {
         for (auto i : {100, 1000, 10000, 0})
         {
@@ -366,9 +368,13 @@ namespace vcpkg
                 .string_arg("--location")
                 .string_arg("-w")
                 .string_arg(Strings::concat(guid_marker, " %{http_code}\\n"));
-            for (auto&& url : url_pairs)
+            for (auto&& url : url_header_path_tuples)
             {
-                cmd.string_arg(url.first).string_arg("-o").string_arg(url.second);
+                cmd.string_arg(std::get<0>(url)).string_arg("-o").string_arg(std::get<2>(url));
+                for (StringView header : std::get<1>(url))
+                {
+                    cmd.string_arg("-H").string_arg(header);
+                }
             }
             auto res = cmd_execute_and_stream_lines(cmd, [out](StringView line) {
                            if (Strings::starts_with(line, guid_marker))
@@ -378,40 +384,42 @@ namespace vcpkg
                        }).value_or_exit(VCPKG_LINE_INFO);
             Checks::check_exit(VCPKG_LINE_INFO, res == 0, "Error: curl failed to execute with exit code: %d", res);
 
-            if (start_size + url_pairs.size() > out->size())
+            if (start_size + url_header_path_tuples.size() > out->size())
             {
                 // curl stopped before finishing all downloads; retry after some time
                 msg::println_warning(msgUnexpectedErrorDuringBulkDownload);
                 std::this_thread::sleep_for(std::chrono::milliseconds(i));
-                url_pairs =
-                    View<std::pair<std::string, Path>>{url_pairs.begin() + out->size() - start_size, url_pairs.end()};
+                url_header_path_tuples = View<std::tuple<std::string, View<std::string>, Path>>{
+                    url_header_path_tuples.begin() + out->size() - start_size, url_header_path_tuples.end()};
             }
             else
             {
                 break;
             }
         }
     }
-    std::vector<int> download_files(Filesystem& fs, View<std::pair<std::string, Path>> url_pairs)
+    std::vector<int> download_files(Filesystem& fs,
+                                    View<std::tuple<std::string, View<std::string>, Path>> url_header_path_tuples)
     {
         static constexpr size_t batch_size = 50;
 
         std::vector<int> ret;
 
         size_t i = 0;
-        for (; i + batch_size <= url_pairs.size(); i += batch_size)
+        for (; i + batch_size <= url_header_path_tuples.size(); i += batch_size)
         {
-            download_files_inner(fs, {url_pairs.data() + i, batch_size}, &ret);
+            download_files_inner(fs, {url_header_path_tuples.data() + i, batch_size}, &ret);
         }
-        if (i != url_pairs.size()) download_files_inner(fs, {url_pairs.begin() + i, url_pairs.end()}, &ret);
+        if (i != url_header_path_tuples.size())
+            download_files_inner(fs, {url_header_path_tuples.begin() + i, url_header_path_tuples.end()}, &ret);
 
         Checks::check_exit(
             VCPKG_LINE_INFO,
-            ret.size() == url_pairs.size(),
+            ret.size() == url_header_path_tuples.size(),
             "Error: curl returned a different number of response codes than were expected for the request (",
             ret.size(),
             " vs expected ",
-            url_pairs.size(),
+            url_header_path_tuples.size(),
             ")");
         return ret;
     }

src/vcpkg/base/strings.cpp

@@ -1,6 +1,7 @@
 #include <vcpkg/base/api_stable_format.h>
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/expected.h>
+#include <vcpkg/base/messages.h>
 #include <vcpkg/base/parse.h>
 #include <vcpkg/base/strings.h>
 #include <vcpkg/base/unicode.h>
@@ -15,7 +16,18 @@
 #include <string>
 #include <vector>
 
-vcpkg::ExpectedS<std::string> vcpkg::details::api_stable_format_impl(StringView sv,
+using namespace vcpkg;
+
+namespace
+{
+
+    DECLARE_AND_REGISTER_MESSAGE(InvalidFormatString,
+                                 (msg::actual),
+                                 "{actual} is the provided format string",
+                                 "invalid format string: {actual}");
+}
+
+vcpkg::ExpectedL<std::string> vcpkg::details::api_stable_format_impl(StringView sv,
                                                                      void (*cb)(void*, std::string&, StringView),
                                                                      void* user)
 {
@@ -37,7 +49,7 @@ vcpkg::ExpectedS<std::string> vcpkg::details::api_stable_format_impl(StringView
         {
             if (p == last)
             {
-                return {Strings::concat("Error: invalid format string: ", sv), expected_right_tag};
+                return msg::format(msgInvalidFormatString, msg::actual = sv);
             }
             else if (*p == '{')
             {
@@ -51,7 +63,7 @@ vcpkg::ExpectedS<std::string> vcpkg::details::api_stable_format_impl(StringView
                 p = std::find_first_of(p, last, s_brackets, s_brackets + 2);
                 if (p == last || p[0] != '}')
                 {
-                    return {Strings::concat("Error: invalid format string: ", sv), expected_right_tag};
+                    return msg::format(msgInvalidFormatString, msg::actual = sv);
                 }
                 // p[0] == '}'
                 cb(user, out, {seq_start, p});
@@ -62,7 +74,7 @@ vcpkg::ExpectedS<std::string> vcpkg::details::api_stable_format_impl(StringView
         {
             if (p == last || p[0] != '}')
             {
-                return {Strings::concat("Error: invalid format string: ", sv), expected_right_tag};
+                return msg::format(msgInvalidFormatString, msg::actual = sv);
             }
             out.push_back('}');
             prev = ++p;