Coverage Report

Created: 2026-10-27 19:19

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/service/http/http_client.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <curl/curl.h>
21
#include <curl/system.h>
22
#include <stdint.h>
23
24
#include <cstdio>
25
#include <functional>
26
#include <string>
27
#include <unordered_set>
28
29
#include "common/status.h"
30
#include "service/http/http_headers.h"
31
#include "service/http/http_method.h"
32
33
namespace doris {
34
35
struct HttpTransferMetrics {
36
    int64_t new_connections = 0;
37
    int64_t dns_time_ns = 0;
38
    int64_t connect_time_ns = 0;
39
    int64_t tls_time_ns = 0;
40
    int64_t first_byte_time_ns = 0;
41
    int64_t total_time_ns = 0;
42
};
43
44
// Helper class to access HTTP resource
45
class HttpClient {
46
public:
47
    enum class AuthTokenMode { CLUSTER, NONE };
48
49
    HttpClient();
50
    ~HttpClient();
51
52
    // you can call this function to execute HTTP request with retry,
53
    // if callback return OK, this function will end and return OK.
54
    // This function will return FAIL if three are more than retry_times
55
    // that callback return FAIL.
56
    static Status execute_with_retry(int retry_times, int sleep_time,
57
                                     const std::function<Status(HttpClient*)>& callback);
58
59
    // this function must call before other function,
60
    // you can call this multiple times to reuse this object
61
    Status init(const std::string& url, bool set_fail_on_error = true,
62
                AuthTokenMode auth_token_mode = AuthTokenMode::CLUSTER);
63
64
    // Release request-owned options and callbacks while retaining reusable connections.
65
    void reset();
66
67
    void set_method(HttpMethod method);
68
69
35
    void set_basic_auth(const std::string& user, const std::string& passwd) {
70
35
        curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
71
35
        curl_easy_setopt(_curl, CURLOPT_USERNAME, user.c_str());
72
35
        curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str());
73
35
    }
74
75
    // Auth-Token: xxxx
76
509
    void set_auth_token(const std::string& token) {
77
509
        std::string scratch_str = HttpHeaders::AUTH_TOKEN + ": " + token;
78
509
        _header_list = curl_slist_append(_header_list, scratch_str.c_str());
79
509
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
80
509
    }
81
82
    // Authorization: xxxx
83
26
    void set_authorization(const std::string& auth) {
84
26
        std::string scratch_str = std::string(HttpHeaders::AUTHORIZATION) + ": " + auth;
85
26
        _header_list = curl_slist_append(_header_list, scratch_str.c_str());
86
26
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
87
26
    }
88
89
    // content_type such as "application/json"
90
372
    void set_content_type(const std::string content_type) {
91
372
        std::string scratch_str = "Content-Type: " + content_type;
92
372
        _header_list = curl_slist_append(_header_list, scratch_str.c_str());
93
372
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
94
372
    }
95
96
363
    void set_payload(const std::string& post_body) {
97
363
        curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length());
98
363
        curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str());
99
363
    }
100
101
    // Override the trusted CA bundle after init(), which resets curl options.
102
    Status set_ca_cert_file(const std::string& path);
103
104
0
    void use_untrusted_ssl() {
105
0
        curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L);
106
0
        curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
107
0
    }
108
109
    void set_speed_limit();
110
111
    // key: value
112
480
    void set_header(const std::string& key, const std::string& value) {
113
480
        std::string header = key + ": " + value;
114
480
        _header_list = curl_slist_append(_header_list, header.c_str());
115
480
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
116
480
    }
117
118
0
    std::string get_response_content_type() {
119
0
        char* ct = nullptr;
120
0
        auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ct);
121
0
        if (code == CURLE_OK && ct != nullptr) {
122
0
            return ct;
123
0
        }
124
0
        return std::string();
125
0
    }
126
127
    // Set the long gohead parameter to 1L to continue send authentication (user+password)
128
    // credentials when following locations, even when hostname changed.
129
0
    void set_unrestricted_auth(int gohead) {
130
0
        curl_easy_setopt(_curl, CURLOPT_UNRESTRICTED_AUTH, gohead);
131
0
    }
132
133
407
    void set_timeout_ms(int64_t timeout_ms) {
134
407
        curl_easy_setopt(_curl, CURLOPT_TIMEOUT_MS, timeout_ms);
135
407
    }
136
137
    // Register a callback that libcurl polls while the request is in flight: often while
138
    // data is flowing, and about once per second when the connection is idle. Returning
139
    // true aborts the transfer right away, so `execute()` fails instead of blocking until
140
    // CURLOPT_TIMEOUT_MS expires.
141
    //
142
    // This is for callers that must be able to give up on a request which may never be
143
    // answered, e.g. a background worker that is being stopped while the http service
144
    // serving its request is going away. Must be called after init(), which resets all
145
    // curl options. Passing an empty callback clears a previously registered one.
146
    void set_abort_callback(std::function<bool()> callback);
147
148
    // used to get content length
149
    // return -1 as error
150
6
    Status get_content_length(uint64_t* length) const {
151
6
        curl_off_t cl;
152
6
        auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl);
153
6
        if (!code) {
154
6
            if (cl < 0) {
155
0
                return Status::InternalError(
156
0
                        fmt::format("failed to get content length, it should be a positive value, "
157
0
                                    "actual is : {}",
158
0
                                    cl));
159
0
            }
160
6
            *length = (uint64_t)cl;
161
6
            return Status::OK();
162
6
        }
163
0
        return Status::InternalError("failed to get content length. err code: {}", code);
164
6
    }
165
166
    // Get the value of the header CONTENT-MD5. The output is empty if no such header exists.
167
    Status get_content_md5(std::string* md5) const;
168
169
278
    CURLcode get_last_curl_code() const { return _last_curl_code; }
170
171
    // Metrics for the latest transfer, including failed requests. Read before reset/init.
172
    // DNS/first-byte/total are cumulative curl timings; connect/TLS are phase durations.
173
    HttpTransferMetrics get_transfer_metrics() const;
174
175
    // Read the final response header; clear value when the header is absent.
176
    Status get_response_header(const char* name, std::string* value) const;
177
178
349
    long get_http_status() const {
179
349
        long code;
180
349
        curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &code);
181
349
        return code;
182
349
    }
183
184
    // execute a head method
185
2
    Status head() {
186
2
        set_method(HEAD);
187
2
        return execute();
188
2
    }
189
190
    // helper function to download a file, you can call this function to download
191
    // a file to local_path
192
    Status download(const std::string& local_path);
193
    Status download_multi_files(const std::string& local_dir,
194
                                const std::unordered_set<std::string>& expected_files);
195
196
    Status execute_post_request(const std::string& payload, std::string* response);
197
198
    Status execute_put_request(const std::string& payload, std::string* response);
199
200
    Status execute_delete_request(const std::string& payload, std::string* response);
201
202
    // execute a simple method, and its response is saved in response argument
203
    Status execute(std::string* response);
204
205
    // execute remote call action
206
    Status execute(const std::function<bool(const void* data, size_t length)>& callback = {});
207
208
    // execute remote call action with retry, like execute_with_retry but keep the http client instance
209
    Status execute(int retry_times, int sleep_time,
210
                   const std::function<Status(HttpClient*)>& callback);
211
212
    size_t on_response_data(const void* data, size_t length);
213
214
    // The file name of the variant column with the inverted index contains %
215
    // such as: 020000000000003f624c4c322c568271060f9b5b274a4a95_0_10133@properties%2Emessage.idx
216
    //  {rowset_id}_{seg_num}_{index_id}_{variant_column_name}{%2E}{extracted_column_name}.idx
217
    // We need to handle %, otherwise it will cause an HTTP 404 error.
218
    // Because the percent ("%") character serves as the indicator for percent-encoded octets,
219
    // it must be percent-encoded as "%25" for that octet to be used as data within a URI.
220
    // https://datatracker.ietf.org/doc/html/rfc3986
221
    Status _escape_url(const std::string& url, std::string* escaped_url);
222
223
2
    void set_range(size_t offset, size_t length) {
224
2
        std::string range_header = "Range: bytes=" + std::to_string(offset) + "-" +
225
2
                                   std::to_string(offset + length - 1);
226
2
        _header_list = curl_slist_append(_header_list, range_header.c_str());
227
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
228
2
    }
229
230
private:
231
    static std::string _get_ca_certificate_path(const std::string& configured_paths);
232
    Status _set_ca_certificate_from_config();
233
    const char* _to_errmsg(CURLcode code) const;
234
    const char* _get_url() const;
235
236
private:
237
    CURL* _curl = nullptr;
238
    CURLcode _last_curl_code = CURLE_OK;
239
    using HttpCallback = std::function<bool(const void* data, size_t length)>;
240
    const HttpCallback* _callback = nullptr;
241
    std::function<bool()> _abort_callback;
242
    char _error_buf[CURL_ERROR_SIZE];
243
    curl_slist* _header_list = nullptr;
244
    HttpMethod _method = GET;
245
};
246
247
} // namespace doris