/root/doris/be/src/http/utils.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "http/utils.h" |
19 | | |
20 | | #include <fcntl.h> |
21 | | #include <stdint.h> |
22 | | #include <sys/stat.h> |
23 | | #include <unistd.h> |
24 | | |
25 | | #include <ostream> |
26 | | #include <vector> |
27 | | |
28 | | #include "common/config.h" |
29 | | #include "common/logging.h" |
30 | | #include "common/status.h" |
31 | | #include "common/utils.h" |
32 | | #include "http/http_channel.h" |
33 | | #include "http/http_common.h" |
34 | | #include "http/http_headers.h" |
35 | | #include "http/http_method.h" |
36 | | #include "http/http_request.h" |
37 | | #include "http/http_status.h" |
38 | | #include "io/fs/file_system.h" |
39 | | #include "io/fs/local_file_system.h" |
40 | | #include "olap/wal/wal_manager.h" |
41 | | #include "runtime/exec_env.h" |
42 | | #include "util/md5.h" |
43 | | #include "util/path_util.h" |
44 | | #include "util/url_coding.h" |
45 | | |
46 | | namespace doris { |
47 | | |
48 | 1 | std::string encode_basic_auth(const std::string& user, const std::string& passwd) { |
49 | 1 | std::string auth = user + ":" + passwd; |
50 | 1 | std::string encoded_auth; |
51 | 1 | base64_encode(auth, &encoded_auth); |
52 | 1 | static std::string s_prefix = "Basic "; |
53 | 1 | return s_prefix + encoded_auth; |
54 | 1 | } |
55 | | |
56 | 39 | bool parse_basic_auth(const HttpRequest& req, std::string* user, std::string* passwd) { |
57 | | // const auto& token = req.header(HttpHeaders::AUTH_TOKEN); |
58 | | |
59 | 39 | const char k_basic[] = "Basic "; |
60 | 39 | const auto& auth = req.header(HttpHeaders::AUTHORIZATION); |
61 | 39 | if (auth.compare(0, sizeof(k_basic) - 1, k_basic, sizeof(k_basic) - 1) != 0) { |
62 | 15 | return false; |
63 | 15 | } |
64 | 24 | std::string encoded_str = auth.substr(sizeof(k_basic) - 1); |
65 | 24 | std::string decoded_auth; |
66 | 24 | if (!base64_decode(encoded_str, &decoded_auth)) { |
67 | 2 | return false; |
68 | 2 | } |
69 | 22 | auto pos = decoded_auth.find(':'); |
70 | 22 | if (pos == std::string::npos) { |
71 | 0 | return false; |
72 | 0 | } |
73 | 22 | user->assign(decoded_auth.c_str(), pos); |
74 | 22 | passwd->assign(decoded_auth.c_str() + pos + 1); |
75 | | |
76 | 22 | return true; |
77 | 22 | } |
78 | | |
79 | 28 | bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth) { |
80 | | // deprecated, removed in 3.1, use AUTH_TOKEN |
81 | 28 | const auto& token = req.header("token"); |
82 | | // deprecated, removed in 3.1, use AUTH_TOKEN |
83 | 28 | const auto& auth_code = req.header(HTTP_AUTH_CODE); |
84 | 28 | const auto& auth_token = req.header(HttpHeaders::AUTH_TOKEN); |
85 | | |
86 | 28 | std::tuple<std::string, std::string, std::string> tmp; |
87 | 28 | auto& [user, pass, cluster] = tmp; |
88 | 28 | bool valid_basic_auth = parse_basic_auth(req, &user, &pass); |
89 | 28 | if (valid_basic_auth) { // always set the basic auth, the user may be useful |
90 | 15 | auto pos = user.find('@'); |
91 | 15 | if (pos != std::string::npos) { |
92 | 0 | cluster.assign(user.c_str() + pos + 1); |
93 | 0 | user.assign(user.c_str(), pos); // user is updated |
94 | 0 | } |
95 | 15 | auth->user = user; |
96 | 15 | auth->passwd = pass; |
97 | 15 | auth->cluster = cluster; |
98 | 15 | } |
99 | | |
100 | 28 | if (!token.empty()) { |
101 | 0 | auth->token = token; // deprecated |
102 | 28 | } else if (!auth_token.empty()) { |
103 | 2 | auth->token = auth_token; |
104 | 26 | } else if (!auth_code.empty()) { |
105 | 0 | auth->auth_code = std::stoll(auth_code); // deprecated |
106 | 26 | } else if (!valid_basic_auth) { |
107 | 11 | return false; |
108 | 11 | } |
109 | | |
110 | | // set user ip |
111 | 17 | auth->user_ip.assign(req.remote_host() != nullptr ? req.remote_host() : ""); |
112 | | |
113 | 17 | return true; |
114 | 28 | } |
115 | | |
116 | | // Do a simple decision, only deal a few type |
117 | 1 | std::string get_content_type(const std::string& file_name) { |
118 | 1 | std::string file_ext = path_util::file_extension(file_name); |
119 | 1 | VLOG_TRACE << "file_name: " << file_name << "; file extension: [" << file_ext << "]"; |
120 | 1 | if (file_ext == std::string(".html") || file_ext == std::string(".htm")) { |
121 | 0 | return "text/html; charset=utf-8"; |
122 | 1 | } else if (file_ext == std::string(".js")) { |
123 | 0 | return "application/javascript; charset=utf-8"; |
124 | 1 | } else if (file_ext == std::string(".css")) { |
125 | 0 | return "text/css; charset=utf-8"; |
126 | 1 | } else if (file_ext == std::string(".txt")) { |
127 | 0 | return "text/plain; charset=utf-8"; |
128 | 1 | } else if (file_ext == std::string(".png")) { |
129 | 0 | return "image/png"; |
130 | 1 | } else if (file_ext == std::string(".ico")) { |
131 | 0 | return "image/x-icon"; |
132 | 1 | } else { |
133 | 1 | return "text/plain; charset=utf-8"; |
134 | 1 | } |
135 | 1 | } |
136 | | |
137 | | void do_file_response(const std::string& file_path, HttpRequest* req, |
138 | 1 | bufferevent_rate_limit_group* rate_limit_group, bool is_acquire_md5) { |
139 | 1 | if (file_path.find("..") != std::string::npos) { |
140 | 0 | LOG(WARNING) << "Not allowed to read relative path: " << file_path; |
141 | 0 | HttpChannel::send_error(req, HttpStatus::FORBIDDEN); |
142 | 0 | return; |
143 | 0 | } |
144 | | |
145 | | // read file content and send response |
146 | 1 | int fd = open(file_path.c_str(), O_RDONLY); |
147 | 1 | if (fd < 0) { |
148 | 0 | LOG(WARNING) << "Failed to open file: " << file_path; |
149 | 0 | HttpChannel::send_error(req, HttpStatus::NOT_FOUND); |
150 | 0 | return; |
151 | 0 | } |
152 | 1 | struct stat st; |
153 | 1 | auto res = fstat(fd, &st); |
154 | 1 | if (res < 0) { |
155 | 0 | close(fd); |
156 | 0 | LOG(WARNING) << "Failed to open file: " << file_path; |
157 | 0 | HttpChannel::send_error(req, HttpStatus::NOT_FOUND); |
158 | 0 | return; |
159 | 0 | } |
160 | | |
161 | 1 | int64_t file_size = st.st_size; |
162 | | |
163 | | // TODO(lingbin): process "IF_MODIFIED_SINCE" header |
164 | | // TODO(lingbin): process "RANGE" header |
165 | 1 | const std::string& range_header = req->header(HttpHeaders::RANGE); |
166 | 1 | if (!range_header.empty()) { |
167 | | // analyse range header |
168 | 0 | } |
169 | | |
170 | 1 | req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str()); |
171 | | |
172 | 1 | if (is_acquire_md5) { |
173 | 1 | Md5Digest md5; |
174 | | |
175 | 1 | void* buf = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); |
176 | 1 | md5.update(buf, file_size); |
177 | 1 | md5.digest(); |
178 | 1 | munmap(buf, file_size); |
179 | | |
180 | 1 | req->add_output_header(HttpHeaders::CONTENT_MD5, md5.hex().c_str()); |
181 | 1 | } |
182 | | |
183 | 1 | if (req->method() == HttpMethod::HEAD) { |
184 | 1 | close(fd); |
185 | 1 | req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str()); |
186 | 1 | HttpChannel::send_reply(req); |
187 | 1 | return; |
188 | 1 | } |
189 | | |
190 | 0 | HttpChannel::send_file(req, fd, 0, file_size, rate_limit_group); |
191 | 0 | } |
192 | | |
193 | 0 | void do_dir_response(const std::string& dir_path, HttpRequest* req) { |
194 | 0 | bool exists = true; |
195 | 0 | std::vector<io::FileInfo> files; |
196 | 0 | Status st = io::global_local_filesystem()->list(dir_path, true, &files, &exists); |
197 | 0 | if (!st.ok()) { |
198 | 0 | LOG(WARNING) << "Failed to scan dir. " << st; |
199 | 0 | HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR); |
200 | 0 | } |
201 | |
|
202 | 0 | const std::string FILE_DELIMITER_IN_DIR_RESPONSE = "\n"; |
203 | |
|
204 | 0 | std::stringstream result; |
205 | 0 | for (auto& file : files) { |
206 | 0 | result << file.file_name << FILE_DELIMITER_IN_DIR_RESPONSE; |
207 | 0 | } |
208 | |
|
209 | 0 | std::string result_str = result.str(); |
210 | 0 | HttpChannel::send_reply(req, result_str); |
211 | 0 | } |
212 | | |
213 | 2 | bool load_size_smaller_than_wal_limit(int64_t content_length) { |
214 | | // 1. req->header(HttpHeaders::CONTENT_LENGTH) will return streamload content length. If it is empty or equals to 0, it means this streamload |
215 | | // is a chunked streamload and we are not sure its size. |
216 | | // 2. if streamload content length is too large, like larger than 80% of the WAL constrain. |
217 | | // |
218 | | // This two cases, we are not certain that the Write-Ahead Logging (WAL) constraints allow for writing down |
219 | | // these blocks within the limited space. So we need to set group_commit = false to avoid dead lock. |
220 | 2 | size_t max_available_size = ExecEnv::GetInstance()->wal_mgr()->get_max_available_size(); |
221 | 2 | return (content_length < 0.8 * max_available_size); |
222 | 2 | } |
223 | | |
224 | | } // namespace doris |