Coverage Report

Created: 2024-11-21 11:46

/root/doris/be/src/http/http_client.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <curl/curl.h>
21
#include <curl/system.h>
22
#include <stdint.h>
23
24
#include <cstdio>
25
#include <functional>
26
#include <string>
27
28
#include "common/status.h"
29
#include "http/http_headers.h"
30
#include "http/http_method.h"
31
32
namespace doris {
33
34
// Helper class to access HTTP resource
35
class HttpClient {
36
public:
37
    HttpClient();
38
    ~HttpClient();
39
40
    // you can call this function to execute HTTP request with retry,
41
    // if callback return OK, this function will end and return OK.
42
    // This function will return FAIL if three are more than retry_times
43
    // that callback return FAIL.
44
    static Status execute_with_retry(int retry_times, int sleep_time,
45
                                     const std::function<Status(HttpClient*)>& callback);
46
47
    // this function must call before other function,
48
    // you can call this multiple times to reuse this object
49
    Status init(const std::string& url, bool set_fail_on_error = true);
50
51
    void set_method(HttpMethod method);
52
53
21
    void set_basic_auth(const std::string& user, const std::string& passwd) {
54
21
        curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
55
21
        curl_easy_setopt(_curl, CURLOPT_USERNAME, user.c_str());
56
21
        curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str());
57
21
    }
58
59
    // Auth-Token: xxxx
60
2
    void set_auth_token(const std::string& token) {
61
2
        std::string scratch_str = HttpHeaders::AUTH_TOKEN + ": " + token;
62
2
        _header_list = curl_slist_append(_header_list, scratch_str.c_str());
63
2
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
64
2
    }
65
66
    // content_type such as "application/json"
67
0
    void set_content_type(const std::string content_type) {
68
0
        std::string scratch_str = "Content-Type: " + content_type;
69
0
        _header_list = curl_slist_append(_header_list, scratch_str.c_str());
70
0
        curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list);
71
0
    }
72
73
11
    void set_payload(const std::string& post_body) {
74
11
        curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length());
75
11
        curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str());
76
11
    }
77
78
    // Currently, only fake SSL configurations are supported
79
0
    void use_untrusted_ssl() {
80
0
        curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L);
81
0
        curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
82
0
    }
83
84
    // TODO(zc): support set header
85
    // void set_header(const std::string& key, const std::string& value) {
86
    // _cntl.http_request().SetHeader(key, value);
87
    // }
88
89
0
    std::string get_response_content_type() {
90
0
        char* ct = nullptr;
91
0
        auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ct);
92
0
        if (code == CURLE_OK && ct != nullptr) {
93
0
            return ct;
94
0
        }
95
0
        return std::string();
96
0
    }
97
98
    // Set the long gohead parameter to 1L to continue send authentication (user+password)
99
    // credentials when following locations, even when hostname changed.
100
0
    void set_unrestricted_auth(int gohead) {
101
0
        curl_easy_setopt(_curl, CURLOPT_UNRESTRICTED_AUTH, gohead);
102
0
    }
103
104
16
    void set_timeout_ms(int64_t timeout_ms) {
105
16
        curl_easy_setopt(_curl, CURLOPT_TIMEOUT_MS, timeout_ms);
106
16
    }
107
108
    // used to get content length
109
    // return -1 as error
110
3
    Status get_content_length(uint64_t* length) const {
111
3
        curl_off_t cl;
112
3
        auto code = curl_easy_getinfo(_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl);
113
3
        if (!code) {
114
3
            if (cl < 0) {
115
0
                return Status::InternalError(
116
0
                        fmt::format("failed to get content length, it should be a positive value, "
117
0
                                    "actual is : {}",
118
0
                                    cl));
119
0
            }
120
3
            *length = (uint64_t)cl;
121
3
            return Status::OK();
122
3
        }
123
0
        return Status::InternalError("failed to get content length. err code: {}", code);
124
3
    }
125
126
    // Get the value of the header CONTENT-MD5. The output is empty if no such header exists.
127
    Status get_content_md5(std::string* md5) const;
128
129
0
    long get_http_status() const {
130
0
        long code;
131
0
        curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &code);
132
0
        return code;
133
0
    }
134
135
    // execute a head method
136
0
    Status head() {
137
0
        set_method(HEAD);
138
0
        return execute();
139
0
    }
140
141
    // helper function to download a file, you can call this function to download
142
    // a file to local_path
143
    Status download(const std::string& local_path);
144
145
    Status execute_post_request(const std::string& payload, std::string* response);
146
147
    Status execute_delete_request(const std::string& payload, std::string* response);
148
149
    // execute a simple method, and its response is saved in response argument
150
    Status execute(std::string* response);
151
152
    // execute remote call action
153
    Status execute(const std::function<bool(const void* data, size_t length)>& callback = {});
154
155
    size_t on_response_data(const void* data, size_t length);
156
157
    // The file name of the variant column with the inverted index contains %
158
    // such as: 020000000000003f624c4c322c568271060f9b5b274a4a95_0_10133@properties%2Emessage.idx
159
    //  {rowset_id}_{seg_num}_{index_id}_{variant_column_name}{%2E}{extracted_column_name}.idx
160
    // We need to handle %, otherwise it will cause an HTTP 404 error.
161
    // Because the percent ("%") character serves as the indicator for percent-encoded octets,
162
    // it must be percent-encoded as "%25" for that octet to be used as data within a URI.
163
    // https://datatracker.ietf.org/doc/html/rfc3986
164
    Status _escape_url(const std::string& url, std::string* escaped_url);
165
166
private:
167
    const char* _to_errmsg(CURLcode code) const;
168
    const char* _get_url() const;
169
170
private:
171
    CURL* _curl = nullptr;
172
    using HttpCallback = std::function<bool(const void* data, size_t length)>;
173
    const HttpCallback* _callback = nullptr;
174
    char _error_buf[CURL_ERROR_SIZE];
175
    curl_slist* _header_list = nullptr;
176
};
177
178
} // namespace doris