Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <curl/curl.h> |
21 | | #include <curl/system.h> |
22 | | #include <stdint.h> |
23 | | |
24 | | #include <cstdio> |
25 | | #include <functional> |
26 | | #include <string> |
27 | | #include <unordered_set> |
28 | | |
29 | | #include "common/status.h" |
30 | | #include "service/http/http_headers.h" |
31 | | #include "service/http/http_method.h" |
32 | | |
33 | | namespace doris { |
34 | | |
35 | | struct HttpTransferMetrics { |
36 | | int64_t new_connections = 0; |
37 | | int64_t dns_time_ns = 0; |
38 | | int64_t connect_time_ns = 0; |
39 | | int64_t tls_time_ns = 0; |
40 | | int64_t first_byte_time_ns = 0; |
41 | | int64_t total_time_ns = 0; |
42 | | }; |
43 | | |
44 | | // Helper class to access HTTP resource |
45 | | class HttpClient { |
46 | | public: |
47 | | HttpClient(); |
48 | | ~HttpClient(); |
49 | | |
50 | | // you can call this function to execute HTTP request with retry, |
51 | | // if callback return OK, this function will end and return OK. |
52 | | // This function will return FAIL if three are more than retry_times |
53 | | // that callback return FAIL. |
54 | | static Status execute_with_retry(int retry_times, int sleep_time, |
55 | | const std::function<Status(HttpClient*)>& callback); |
56 | | |
57 | | // this function must call before other function, |
58 | | // you can call this multiple times to reuse this object |
59 | | Status init(const std::string& url, bool set_fail_on_error = true); |
60 | | |
61 | | // Release request-owned options and callbacks while retaining reusable connections. |
62 | | void reset(); |
63 | | |
64 | | void set_method(HttpMethod method); |
65 | | |
66 | 21 | void set_basic_auth(const std::string& user, const std::string& passwd) { |
67 | 21 | curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); |
68 | 21 | curl_easy_setopt(_curl, CURLOPT_USERNAME, user.c_str()); |
69 | 21 | curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str()); |
70 | 21 | } |
71 | | |
72 | | // Auth-Token: xxxx |
73 | 505 | void set_auth_token(const std::string& token) { |
74 | 505 | std::string scratch_str = HttpHeaders::AUTH_TOKEN + ": " + token; |
75 | 505 | _header_list = curl_slist_append(_header_list, scratch_str.c_str()); |
76 | 505 | curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); |
77 | 505 | } |
78 | | |
79 | | // Authorization: xxxx |
80 | 20 | void set_authorization(const std::string& auth) { |
81 | 20 | std::string scratch_str = std::string(HttpHeaders::AUTHORIZATION) + ": " + auth; |
82 | 20 | _header_list = curl_slist_append(_header_list, scratch_str.c_str()); |
83 | 20 | curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); |
84 | 20 | } |
85 | | |
86 | | // content_type such as "application/json" |
87 | 353 | void set_content_type(const std::string content_type) { |
88 | 353 | std::string scratch_str = "Content-Type: " + content_type; |
89 | 353 | _header_list = curl_slist_append(_header_list, scratch_str.c_str()); |
90 | 353 | curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); |
91 | 353 | } |
92 | | |
93 | 347 | void set_payload(const std::string& post_body) { |
94 | 347 | curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length()); |
95 | 347 | curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str()); |
96 | 347 | } |
97 | | |
98 | | // Currently, only fake SSL configurations are supported |
99 | 0 | void use_untrusted_ssl() { |
100 | 0 | curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L); |
101 | 0 | curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L); |
102 | 0 | } |
103 | | |
104 | | void set_speed_limit(); |
105 | | |
106 | | // key: value |
107 | 456 | void set_header(const std::string& key, const std::string& value) { |
108 | 456 | std::string header = key + ": " + value; |
109 | 456 | _header_list = curl_slist_append(_header_list, header.c_str()); |
110 | 456 | curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); |
111 | 456 | } |
112 | | |
113 | 0 | std::string get_response_content_type() { |
114 | 0 | char* ct = nullptr; |
115 | 0 | auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ct); |
116 | 0 | if (code == CURLE_OK && ct != nullptr) { |
117 | 0 | return ct; |
118 | 0 | } |
119 | 0 | return std::string(); |
120 | 0 | } |
121 | | |
122 | | // Set the long gohead parameter to 1L to continue send authentication (user+password) |
123 | | // credentials when following locations, even when hostname changed. |
124 | 0 | void set_unrestricted_auth(int gohead) { |
125 | 0 | curl_easy_setopt(_curl, CURLOPT_UNRESTRICTED_AUTH, gohead); |
126 | 0 | } |
127 | | |
128 | 388 | void set_timeout_ms(int64_t timeout_ms) { |
129 | 388 | curl_easy_setopt(_curl, CURLOPT_TIMEOUT_MS, timeout_ms); |
130 | 388 | } |
131 | | |
132 | | // Register a callback that libcurl polls while the request is in flight: often while |
133 | | // data is flowing, and about once per second when the connection is idle. Returning |
134 | | // true aborts the transfer right away, so `execute()` fails instead of blocking until |
135 | | // CURLOPT_TIMEOUT_MS expires. |
136 | | // |
137 | | // This is for callers that must be able to give up on a request which may never be |
138 | | // answered, e.g. a background worker that is being stopped while the http service |
139 | | // serving its request is going away. Must be called after init(), which resets all |
140 | | // curl options. Passing an empty callback clears a previously registered one. |
141 | | void set_abort_callback(std::function<bool()> callback); |
142 | | |
143 | | // used to get content length |
144 | | // return -1 as error |
145 | 6 | Status get_content_length(uint64_t* length) const { |
146 | 6 | curl_off_t cl; |
147 | 6 | auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl); |
148 | 6 | if (!code) { |
149 | 6 | if (cl < 0) { |
150 | 0 | return Status::InternalError( |
151 | 0 | fmt::format("failed to get content length, it should be a positive value, " |
152 | 0 | "actual is : {}", |
153 | 0 | cl)); |
154 | 0 | } |
155 | 6 | *length = (uint64_t)cl; |
156 | 6 | return Status::OK(); |
157 | 6 | } |
158 | 0 | return Status::InternalError("failed to get content length. err code: {}", code); |
159 | 6 | } |
160 | | |
161 | | // Get the value of the header CONTENT-MD5. The output is empty if no such header exists. |
162 | | Status get_content_md5(std::string* md5) const; |
163 | | |
164 | 278 | CURLcode get_last_curl_code() const { return _last_curl_code; } |
165 | | |
166 | | // Metrics for the latest transfer, including failed requests. Read before reset/init. |
167 | | // DNS/first-byte/total are cumulative curl timings; connect/TLS are phase durations. |
168 | | HttpTransferMetrics get_transfer_metrics() const; |
169 | | |
170 | | // Read the final response header; clear value when the header is absent. |
171 | | Status get_response_header(const char* name, std::string* value) const; |
172 | | |
173 | 332 | long get_http_status() const { |
174 | 332 | long code; |
175 | 332 | curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &code); |
176 | 332 | return code; |
177 | 332 | } |
178 | | |
179 | | // execute a head method |
180 | 2 | Status head() { |
181 | 2 | set_method(HEAD); |
182 | 2 | return execute(); |
183 | 2 | } |
184 | | |
185 | | // helper function to download a file, you can call this function to download |
186 | | // a file to local_path |
187 | | Status download(const std::string& local_path); |
188 | | Status download_multi_files(const std::string& local_dir, |
189 | | const std::unordered_set<std::string>& expected_files); |
190 | | |
191 | | Status execute_post_request(const std::string& payload, std::string* response); |
192 | | |
193 | | Status execute_put_request(const std::string& payload, std::string* response); |
194 | | |
195 | | Status execute_delete_request(const std::string& payload, std::string* response); |
196 | | |
197 | | // execute a simple method, and its response is saved in response argument |
198 | | Status execute(std::string* response); |
199 | | |
200 | | // execute remote call action |
201 | | Status execute(const std::function<bool(const void* data, size_t length)>& callback = {}); |
202 | | |
203 | | // execute remote call action with retry, like execute_with_retry but keep the http client instance |
204 | | Status execute(int retry_times, int sleep_time, |
205 | | const std::function<Status(HttpClient*)>& callback); |
206 | | |
207 | | size_t on_response_data(const void* data, size_t length); |
208 | | |
209 | | // The file name of the variant column with the inverted index contains % |
210 | | // such as: 020000000000003f624c4c322c568271060f9b5b274a4a95_0_10133@properties%2Emessage.idx |
211 | | // {rowset_id}_{seg_num}_{index_id}_{variant_column_name}{%2E}{extracted_column_name}.idx |
212 | | // We need to handle %, otherwise it will cause an HTTP 404 error. |
213 | | // Because the percent ("%") character serves as the indicator for percent-encoded octets, |
214 | | // it must be percent-encoded as "%25" for that octet to be used as data within a URI. |
215 | | // https://datatracker.ietf.org/doc/html/rfc3986 |
216 | | Status _escape_url(const std::string& url, std::string* escaped_url); |
217 | | |
218 | 2 | void set_range(size_t offset, size_t length) { |
219 | 2 | std::string range_header = "Range: bytes=" + std::to_string(offset) + "-" + |
220 | 2 | std::to_string(offset + length - 1); |
221 | 2 | _header_list = curl_slist_append(_header_list, range_header.c_str()); |
222 | | curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); |
223 | 2 | } |
224 | | |
225 | | private: |
226 | | static std::string _get_ca_certificate_path(const std::string& configured_paths); |
227 | | Status _set_ca_certificate_from_config(); |
228 | | const char* _to_errmsg(CURLcode code) const; |
229 | | const char* _get_url() const; |
230 | | |
231 | | private: |
232 | | CURL* _curl = nullptr; |
233 | | CURLcode _last_curl_code = CURLE_OK; |
234 | | using HttpCallback = std::function<bool(const void* data, size_t length)>; |
235 | | const HttpCallback* _callback = nullptr; |
236 | | std::function<bool()> _abort_callback; |
237 | | char _error_buf[CURL_ERROR_SIZE]; |
238 | | curl_slist* _header_list = nullptr; |
239 | | HttpMethod _method = GET; |
240 | | }; |
241 | | |
242 | | } // namespace doris |