Coverage Report

Created: 2024-11-20 15:52

/root/doris/be/src/http/utils.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "http/utils.h"
19
20
#include <fcntl.h>
21
#include <stdint.h>
22
#include <sys/stat.h>
23
#include <unistd.h>
24
25
#include <ostream>
26
#include <vector>
27
28
#include "common/config.h"
29
#include "common/logging.h"
30
#include "common/status.h"
31
#include "common/utils.h"
32
#include "http/http_channel.h"
33
#include "http/http_common.h"
34
#include "http/http_headers.h"
35
#include "http/http_method.h"
36
#include "http/http_request.h"
37
#include "http/http_status.h"
38
#include "io/fs/file_system.h"
39
#include "io/fs/local_file_system.h"
40
#include "olap/wal/wal_manager.h"
41
#include "runtime/exec_env.h"
42
#include "util/md5.h"
43
#include "util/path_util.h"
44
#include "util/url_coding.h"
45
46
namespace doris {
47
48
1
std::string encode_basic_auth(const std::string& user, const std::string& passwd) {
49
1
    std::string auth = user + ":" + passwd;
50
1
    std::string encoded_auth;
51
1
    base64_encode(auth, &encoded_auth);
52
1
    static std::string s_prefix = "Basic ";
53
1
    return s_prefix + encoded_auth;
54
1
}
55
56
39
bool parse_basic_auth(const HttpRequest& req, std::string* user, std::string* passwd) {
57
    // const auto& token = req.header(HttpHeaders::AUTH_TOKEN);
58
59
39
    const char k_basic[] = "Basic ";
60
39
    const auto& auth = req.header(HttpHeaders::AUTHORIZATION);
61
39
    if (auth.compare(0, sizeof(k_basic) - 1, k_basic, sizeof(k_basic) - 1) != 0) {
62
15
        return false;
63
15
    }
64
24
    std::string encoded_str = auth.substr(sizeof(k_basic) - 1);
65
24
    std::string decoded_auth;
66
24
    if (!base64_decode(encoded_str, &decoded_auth)) {
67
2
        return false;
68
2
    }
69
22
    auto pos = decoded_auth.find(':');
70
22
    if (pos == std::string::npos) {
71
0
        return false;
72
0
    }
73
22
    user->assign(decoded_auth.c_str(), pos);
74
22
    passwd->assign(decoded_auth.c_str() + pos + 1);
75
76
22
    return true;
77
22
}
78
79
28
bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth) {
80
    // deprecated, removed in 3.1, use AUTH_TOKEN
81
28
    const auto& token = req.header("token");
82
    // deprecated, removed in 3.1, use AUTH_TOKEN
83
28
    const auto& auth_code = req.header(HTTP_AUTH_CODE);
84
28
    const auto& auth_token = req.header(HttpHeaders::AUTH_TOKEN);
85
86
28
    std::tuple<std::string, std::string, std::string> tmp;
87
28
    auto& [user, pass, cluster] = tmp;
88
28
    bool valid_basic_auth = parse_basic_auth(req, &user, &pass);
89
28
    if (valid_basic_auth) { // always set the basic auth, the user may be useful
90
15
        auto pos = user.find('@');
91
15
        if (pos != std::string::npos) {
92
0
            cluster.assign(user.c_str() + pos + 1);
93
0
            user.assign(user.c_str(), pos); // user is updated
94
0
        }
95
15
        auth->user = user;
96
15
        auth->passwd = pass;
97
15
        auth->cluster = cluster;
98
15
    }
99
100
28
    if (!token.empty()) {
101
0
        auth->token = token; // deprecated
102
28
    } else if (!auth_token.empty()) {
103
2
        auth->token = auth_token;
104
26
    } else if (!auth_code.empty()) {
105
0
        auth->auth_code = std::stoll(auth_code); // deprecated
106
26
    } else if (!valid_basic_auth) {
107
11
        return false;
108
11
    }
109
110
    // set user ip
111
17
    auth->user_ip.assign(req.remote_host() != nullptr ? req.remote_host() : "");
112
113
17
    return true;
114
28
}
115
116
// Do a simple decision, only deal a few type
117
1
std::string get_content_type(const std::string& file_name) {
118
1
    std::string file_ext = path_util::file_extension(file_name);
119
1
    VLOG_TRACE << "file_name: " << file_name << "; file extension: [" << file_ext << "]";
120
1
    if (file_ext == std::string(".html") || file_ext == std::string(".htm")) {
121
0
        return "text/html; charset=utf-8";
122
1
    } else if (file_ext == std::string(".js")) {
123
0
        return "application/javascript; charset=utf-8";
124
1
    } else if (file_ext == std::string(".css")) {
125
0
        return "text/css; charset=utf-8";
126
1
    } else if (file_ext == std::string(".txt")) {
127
0
        return "text/plain; charset=utf-8";
128
1
    } else if (file_ext == std::string(".png")) {
129
0
        return "image/png";
130
1
    } else if (file_ext == std::string(".ico")) {
131
0
        return "image/x-icon";
132
1
    } else {
133
1
        return "text/plain; charset=utf-8";
134
1
    }
135
1
}
136
137
void do_file_response(const std::string& file_path, HttpRequest* req,
138
1
                      bufferevent_rate_limit_group* rate_limit_group, bool is_acquire_md5) {
139
1
    if (file_path.find("..") != std::string::npos) {
140
0
        LOG(WARNING) << "Not allowed to read relative path: " << file_path;
141
0
        HttpChannel::send_error(req, HttpStatus::FORBIDDEN);
142
0
        return;
143
0
    }
144
145
    // read file content and send response
146
1
    int fd = open(file_path.c_str(), O_RDONLY);
147
1
    if (fd < 0) {
148
0
        LOG(WARNING) << "Failed to open file: " << file_path;
149
0
        HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
150
0
        return;
151
0
    }
152
1
    struct stat st;
153
1
    auto res = fstat(fd, &st);
154
1
    if (res < 0) {
155
0
        close(fd);
156
0
        LOG(WARNING) << "Failed to open file: " << file_path;
157
0
        HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
158
0
        return;
159
0
    }
160
161
1
    int64_t file_size = st.st_size;
162
163
    // TODO(lingbin): process "IF_MODIFIED_SINCE" header
164
    // TODO(lingbin): process "RANGE" header
165
1
    const std::string& range_header = req->header(HttpHeaders::RANGE);
166
1
    if (!range_header.empty()) {
167
        // analyse range header
168
0
    }
169
170
1
    req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str());
171
172
1
    if (is_acquire_md5) {
173
1
        Md5Digest md5;
174
175
1
        void* buf = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0);
176
1
        md5.update(buf, file_size);
177
1
        md5.digest();
178
1
        munmap(buf, file_size);
179
180
1
        req->add_output_header(HttpHeaders::CONTENT_MD5, md5.hex().c_str());
181
1
    }
182
183
1
    if (req->method() == HttpMethod::HEAD) {
184
1
        close(fd);
185
1
        req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str());
186
1
        HttpChannel::send_reply(req);
187
1
        return;
188
1
    }
189
190
0
    HttpChannel::send_file(req, fd, 0, file_size, rate_limit_group);
191
0
}
192
193
0
void do_dir_response(const std::string& dir_path, HttpRequest* req) {
194
0
    bool exists = true;
195
0
    std::vector<io::FileInfo> files;
196
0
    Status st = io::global_local_filesystem()->list(dir_path, true, &files, &exists);
197
0
    if (!st.ok()) {
198
0
        LOG(WARNING) << "Failed to scan dir. " << st;
199
0
        HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR);
200
0
    }
201
202
0
    const std::string FILE_DELIMITER_IN_DIR_RESPONSE = "\n";
203
204
0
    std::stringstream result;
205
0
    for (auto& file : files) {
206
0
        result << file.file_name << FILE_DELIMITER_IN_DIR_RESPONSE;
207
0
    }
208
209
0
    std::string result_str = result.str();
210
0
    HttpChannel::send_reply(req, result_str);
211
0
}
212
213
2
bool load_size_smaller_than_wal_limit(int64_t content_length) {
214
    // 1. req->header(HttpHeaders::CONTENT_LENGTH) will return streamload content length. If it is empty or equals to 0, it means this streamload
215
    // is a chunked streamload and we are not sure its size.
216
    // 2. if streamload content length is too large, like larger than 80% of the WAL constrain.
217
    //
218
    // This two cases, we are not certain that the Write-Ahead Logging (WAL) constraints allow for writing down
219
    // these blocks within the limited space. So we need to set group_commit = false to avoid dead lock.
220
2
    size_t max_available_size = ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
221
2
    return (content_length < 0.8 * max_available_size);
222
2
}
223
224
} // namespace doris