common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "cpp/obj-client/auth/gcp/gcs_signed_url.h" |
19 | | |
20 | | #include <fmt/format.h> |
21 | | #include <openssl/sha.h> |
22 | | |
23 | | #include <algorithm> |
24 | | #include <array> |
25 | | #include <ctime> |
26 | | #include <optional> |
27 | | #include <utility> |
28 | | |
29 | | #include "cpp/obj-client/auth/gcp/gcp_auth.h" |
30 | | |
31 | | namespace doris { |
32 | | namespace { |
33 | | |
34 | | constexpr std::string_view SIGNING_ALGORITHM = "GOOG4-RSA-SHA256"; |
35 | | constexpr std::string_view SIGNED_HEADERS = "host"; |
36 | | constexpr std::string_view SIGNING_REGION = "auto"; |
37 | | constexpr std::string_view SIGNING_SERVICE = "storage"; |
38 | | constexpr std::string_view SIGNING_TERMINATOR = "goog4_request"; |
39 | | constexpr int64_t MAX_EXPIRATION_SECONDS = 7 * 24 * 60 * 60; |
40 | | |
41 | | struct Endpoint { |
42 | | std::string scheme; |
43 | | std::string authority; |
44 | | }; |
45 | | |
46 | 912 | bool is_unreserved(unsigned char c) { |
47 | 912 | return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || |
48 | 912 | c == '_' || c == '.' || c == '~'; |
49 | 912 | } |
50 | | |
51 | 27 | std::string percent_encode(std::string_view value, bool preserve_slash) { |
52 | 27 | constexpr char HEX[] = "0123456789ABCDEF"; |
53 | 27 | std::string encoded; |
54 | 27 | encoded.reserve(value.size()); |
55 | 912 | for (unsigned char c : value) { |
56 | 912 | if (is_unreserved(c) || (preserve_slash && c == '/')) { |
57 | 864 | encoded.push_back(static_cast<char>(c)); |
58 | 864 | continue; |
59 | 864 | } |
60 | 48 | encoded.push_back('%'); |
61 | 48 | encoded.push_back(HEX[c >> 4]); |
62 | 48 | encoded.push_back(HEX[c & 0x0F]); |
63 | 48 | } |
64 | 27 | return encoded; |
65 | 27 | } |
66 | | |
67 | 18 | std::string hex_encode(std::string_view value) { |
68 | 18 | constexpr char HEX[] = "0123456789abcdef"; |
69 | 18 | std::string encoded; |
70 | 18 | encoded.reserve(value.size() * 2); |
71 | 343 | for (unsigned char c : value) { |
72 | 343 | encoded.push_back(HEX[c >> 4]); |
73 | 343 | encoded.push_back(HEX[c & 0x0F]); |
74 | 343 | } |
75 | 18 | return encoded; |
76 | 18 | } |
77 | | |
78 | 9 | std::string sha256_hex(std::string_view value) { |
79 | 9 | std::array<unsigned char, SHA256_DIGEST_LENGTH> digest {}; |
80 | 9 | SHA256(reinterpret_cast<const unsigned char*>(value.data()), value.size(), digest.data()); |
81 | 9 | return hex_encode( |
82 | 9 | std::string_view(reinterpret_cast<const char*>(digest.data()), digest.size())); |
83 | 9 | } |
84 | | |
85 | 12 | std::optional<std::string> parse_endpoint(std::string endpoint, Endpoint* parsed) { |
86 | 12 | if (endpoint.empty()) { |
87 | 0 | return "GCS endpoint must not be empty"; |
88 | 0 | } |
89 | 12 | auto scheme_end = endpoint.find("://"); |
90 | 12 | if (scheme_end == std::string::npos) { |
91 | 4 | parsed->scheme = "https"; |
92 | 8 | } else { |
93 | 8 | parsed->scheme = endpoint.substr(0, scheme_end); |
94 | 8 | endpoint.erase(0, scheme_end + 3); |
95 | 8 | } |
96 | 12 | if (!is_valid_gcp_storage_endpoint(parsed->scheme + "://" + endpoint)) { |
97 | 3 | return "GCS signed URL requires a Google Storage HTTPS service-base endpoint without a " |
98 | 3 | "bucket"; |
99 | 3 | } |
100 | 9 | if (endpoint.ends_with('/')) { |
101 | 2 | endpoint.pop_back(); |
102 | 2 | } |
103 | | // HTTP clients omit the default TLS port from Host. Sign the same authority. |
104 | 9 | if (endpoint.ends_with(":443")) { |
105 | 2 | endpoint.resize(endpoint.size() - 4); |
106 | 2 | } |
107 | 9 | parsed->authority = std::move(endpoint); |
108 | 9 | return std::nullopt; |
109 | 12 | } |
110 | | |
111 | | std::optional<std::string> format_signing_time(std::chrono::system_clock::time_point now, |
112 | 9 | std::string* timestamp, std::string* date) { |
113 | 9 | auto time = std::chrono::system_clock::to_time_t(now); |
114 | 9 | std::tm utc {}; |
115 | | #if defined(_WIN32) |
116 | | if (gmtime_s(&utc, &time) != 0) { |
117 | | return "failed to convert GCS signing time to UTC"; |
118 | | } |
119 | | #else |
120 | 9 | if (gmtime_r(&time, &utc) == nullptr) { |
121 | 0 | return "failed to convert GCS signing time to UTC"; |
122 | 0 | } |
123 | 9 | #endif |
124 | 9 | char timestamp_buffer[17]; |
125 | 9 | char date_buffer[9]; |
126 | 9 | if (std::strftime(timestamp_buffer, sizeof(timestamp_buffer), "%Y%m%dT%H%M%SZ", &utc) == 0 || |
127 | 9 | std::strftime(date_buffer, sizeof(date_buffer), "%Y%m%d", &utc) == 0) { |
128 | 0 | return "failed to format GCS signing time"; |
129 | 0 | } |
130 | 9 | *timestamp = timestamp_buffer; |
131 | 9 | *date = date_buffer; |
132 | 9 | return std::nullopt; |
133 | 9 | } |
134 | | |
135 | | } // namespace |
136 | | |
137 | | GcsV4SignedUrlResult build_gcs_v4_signed_url(const GcsV4SignedUrlOptions& options, |
138 | | std::chrono::system_clock::time_point now, |
139 | 13 | const GcsSignBlobFunction& sign_blob) { |
140 | 13 | if (options.bucket.empty()) { |
141 | 0 | return {.error = "GCS bucket must not be empty"}; |
142 | 0 | } |
143 | 13 | if (!is_valid_gcp_service_account_email(options.signer_email)) { |
144 | 0 | return {.error = "invalid GCS signing service account email"}; |
145 | 0 | } |
146 | 13 | if (options.expiration_secs <= 0 || options.expiration_secs > MAX_EXPIRATION_SECONDS) { |
147 | 1 | return {.error = fmt::format("GCS signed URL expiration must be in [1, {}] seconds", |
148 | 1 | MAX_EXPIRATION_SECONDS)}; |
149 | 1 | } |
150 | 12 | if (!sign_blob) { |
151 | 0 | return {.error = "GCS signBlob callback must not be empty"}; |
152 | 0 | } |
153 | | |
154 | 12 | Endpoint endpoint; |
155 | 12 | if (auto error = parse_endpoint(options.endpoint, &endpoint); error.has_value()) { |
156 | 3 | return {.error = std::move(*error)}; |
157 | 3 | } |
158 | 9 | std::string timestamp; |
159 | 9 | std::string date; |
160 | 9 | if (auto error = format_signing_time(now, ×tamp, &date); error.has_value()) { |
161 | 0 | return {.error = std::move(*error)}; |
162 | 0 | } |
163 | | |
164 | 9 | const std::string canonical_uri = |
165 | 9 | "/" + percent_encode(options.bucket, false) + "/" + percent_encode(options.key, true); |
166 | 9 | const std::string credential_scope = |
167 | 9 | fmt::format("{}/{}/{}/{}", date, SIGNING_REGION, SIGNING_SERVICE, SIGNING_TERMINATOR); |
168 | 9 | const std::string credential = options.signer_email + "/" + credential_scope; |
169 | 9 | const std::string canonical_query = fmt::format( |
170 | 9 | "X-Goog-Algorithm={}&X-Goog-Credential={}&X-Goog-Date={}&" |
171 | 9 | "X-Goog-Expires={}&X-Goog-SignedHeaders={}", |
172 | 9 | SIGNING_ALGORITHM, percent_encode(credential, false), timestamp, |
173 | 9 | options.expiration_secs, SIGNED_HEADERS); |
174 | 9 | const std::string canonical_request = |
175 | 9 | fmt::format("GET\n{}\n{}\nhost:{}\n\n{}\nUNSIGNED-PAYLOAD", canonical_uri, |
176 | 9 | canonical_query, endpoint.authority, SIGNED_HEADERS); |
177 | 9 | const std::string string_to_sign = fmt::format("{}\n{}\n{}\n{}", SIGNING_ALGORITHM, timestamp, |
178 | 9 | credential_scope, sha256_hex(canonical_request)); |
179 | | |
180 | 9 | auto sign_result = sign_blob(string_to_sign); |
181 | 9 | if (!sign_result.ok()) { |
182 | 0 | return {.error = std::move(sign_result.error)}; |
183 | 0 | } |
184 | 9 | if (sign_result.signature.empty()) { |
185 | 0 | return {.error = "signBlob returned an empty signature"}; |
186 | 0 | } |
187 | | |
188 | 9 | return {.signed_url = fmt::format("{}://{}{}?{}&X-Goog-Signature={}", endpoint.scheme, |
189 | 9 | endpoint.authority, canonical_uri, canonical_query, |
190 | 9 | hex_encode(sign_result.signature))}; |
191 | 9 | } |
192 | | |
193 | | } // namespace doris |