/root/doris/be/src/storage/utils.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "storage/utils.h" |
19 | | |
20 | | // IWYU pragma: no_include <bthread/errno.h> |
21 | | #include <errno.h> // IWYU pragma: keep |
22 | | #include <stdarg.h> |
23 | | #include <time.h> |
24 | | #include <unistd.h> |
25 | | #include <zconf.h> |
26 | | #include <zlib.h> |
27 | | |
28 | | #include <cstring> |
29 | | #include <memory> |
30 | | #include <regex> |
31 | | #include <set> |
32 | | #include <sstream> |
33 | | #include <string> |
34 | | #include <vector> |
35 | | |
36 | | #include "common/check.h" |
37 | | #include "common/logging.h" |
38 | | #include "common/status.h" |
39 | | #include "core/value/ipv4_value.h" |
40 | | #include "core/value/ipv6_value.h" |
41 | | #include "core/value/uuid_value.h" |
42 | | #include "io/fs/file_reader.h" |
43 | | #include "io/fs/file_writer.h" |
44 | | #include "io/fs/local_file_system.h" |
45 | | #include "storage/olap_common.h" |
46 | | #include "util/sse_util.hpp" |
47 | | #include "util/string_parser.hpp" |
48 | | |
49 | | namespace doris { |
50 | | #include "common/compile_check_begin.h" |
51 | | using namespace ErrorCode; |
52 | | |
53 | 148 | uint32_t olap_adler32_init() { |
54 | 148 | return (uint32_t)adler32(0, Z_NULL, 0); |
55 | 148 | } |
56 | | |
57 | 148 | uint32_t olap_adler32(uint32_t adler, const char* buf, size_t len) { |
58 | 148 | return (uint32_t)adler32(adler, reinterpret_cast<const Bytef*>(buf), (uint32_t)len); |
59 | 148 | } |
60 | | |
61 | 2 | Status gen_timestamp_string(std::string* out_string) { |
62 | 2 | time_t now = time(nullptr); |
63 | 2 | tm local_tm; |
64 | | |
65 | 2 | if (localtime_r(&now, &local_tm) == nullptr) { |
66 | 0 | return Status::Error<OS_ERROR>("fail to localtime_r time. time={}", now); |
67 | 0 | } |
68 | 2 | char time_suffix[16] = {0}; // Example: 20150706111404's length is 15 |
69 | 2 | if (strftime(time_suffix, sizeof(time_suffix), "%Y%m%d%H%M%S", &local_tm) == 0) { |
70 | 0 | return Status::Error<OS_ERROR>("fail to strftime time. time={}", now); |
71 | 0 | } |
72 | | |
73 | 2 | *out_string = time_suffix; |
74 | 2 | return Status::OK(); |
75 | 2 | } |
76 | | |
77 | 0 | Status read_write_test_file(const std::string& test_file_path) { |
78 | 0 | if (access(test_file_path.c_str(), F_OK) == 0) { |
79 | 0 | if (remove(test_file_path.c_str()) != 0) { |
80 | 0 | char errmsg[64]; |
81 | 0 | return Status::IOError("fail to access test file. path={}, errno={}, err={}", |
82 | 0 | test_file_path, errno, strerror_r(errno, errmsg, 64)); |
83 | 0 | } |
84 | 0 | } else { |
85 | 0 | if (errno != ENOENT) { |
86 | 0 | char errmsg[64]; |
87 | 0 | return Status::IOError("fail to access test file. path={}, errno={}, err={}", |
88 | 0 | test_file_path, errno, strerror_r(errno, errmsg, 64)); |
89 | 0 | } |
90 | 0 | } |
91 | | |
92 | 0 | const size_t TEST_FILE_BUF_SIZE = 4096; |
93 | 0 | const size_t DIRECT_IO_ALIGNMENT = 512; |
94 | 0 | char* write_test_buff = nullptr; |
95 | 0 | char* read_test_buff = nullptr; |
96 | 0 | if (posix_memalign((void**)&write_test_buff, DIRECT_IO_ALIGNMENT, TEST_FILE_BUF_SIZE) != 0) { |
97 | 0 | return Status::Error<MEM_ALLOC_FAILED>("fail to allocate write buffer memory. size={}", |
98 | 0 | TEST_FILE_BUF_SIZE); |
99 | 0 | } |
100 | 0 | std::unique_ptr<char, decltype(&std::free)> write_buff(write_test_buff, &std::free); |
101 | 0 | if (posix_memalign((void**)&read_test_buff, DIRECT_IO_ALIGNMENT, TEST_FILE_BUF_SIZE) != 0) { |
102 | 0 | return Status::Error<MEM_ALLOC_FAILED>("fail to allocate read buffer memory. size={}", |
103 | 0 | TEST_FILE_BUF_SIZE); |
104 | 0 | } |
105 | 0 | std::unique_ptr<char, decltype(&std::free)> read_buff(read_test_buff, &std::free); |
106 | | // generate random numbers |
107 | 0 | uint32_t rand_seed = static_cast<uint32_t>(time(nullptr)); |
108 | 0 | for (size_t i = 0; i < TEST_FILE_BUF_SIZE; ++i) { |
109 | 0 | int32_t tmp_value = rand_r(&rand_seed); |
110 | 0 | write_test_buff[i] = static_cast<char>(tmp_value); |
111 | 0 | } |
112 | | |
113 | | // write file |
114 | 0 | io::FileWriterPtr file_writer; |
115 | 0 | RETURN_IF_ERROR(io::global_local_filesystem()->create_file(test_file_path, &file_writer)); |
116 | 0 | RETURN_IF_ERROR(file_writer->append({write_buff.get(), TEST_FILE_BUF_SIZE})); |
117 | 0 | RETURN_IF_ERROR(file_writer->close()); |
118 | | // read file |
119 | 0 | io::FileReaderSPtr file_reader; |
120 | 0 | RETURN_IF_ERROR(io::global_local_filesystem()->open_file(test_file_path, &file_reader)); |
121 | 0 | size_t bytes_read = 0; |
122 | 0 | RETURN_IF_ERROR(file_reader->read_at(0, {read_buff.get(), TEST_FILE_BUF_SIZE}, &bytes_read)); |
123 | 0 | if (memcmp(write_buff.get(), read_buff.get(), TEST_FILE_BUF_SIZE) != 0) { |
124 | 0 | return Status::IOError("the test file write_buf and read_buf not equal, file_name={}.", |
125 | 0 | test_file_path); |
126 | 0 | } |
127 | | // delete file |
128 | 0 | return io::global_local_filesystem()->delete_file(test_file_path); |
129 | 0 | } |
130 | | |
131 | 0 | Status check_datapath_rw(const std::string& path) { |
132 | 0 | bool exists = true; |
133 | 0 | RETURN_IF_ERROR(io::global_local_filesystem()->exists(path, &exists)); |
134 | 0 | if (!exists) { |
135 | 0 | return Status::IOError("path does not exist: {}", path); |
136 | 0 | } |
137 | 0 | std::string file_path = path + "/.read_write_test_file"; |
138 | 0 | return read_write_test_file(file_path); |
139 | 0 | } |
140 | | |
141 | | __thread char Errno::_buf[BUF_SIZE]; ///< buffer instance |
142 | | |
143 | 0 | const char* Errno::str() { |
144 | 0 | return str(no()); |
145 | 0 | } |
146 | | |
147 | 0 | const char* Errno::str(int no) { |
148 | 0 | if (0 != strerror_r(no, _buf, BUF_SIZE)) { |
149 | 0 | LOG(WARNING) << "fail to get errno string. [no='" << no << "', errno='" << errno << "']"; |
150 | 0 | snprintf(_buf, BUF_SIZE, "unknown errno"); |
151 | 0 | } |
152 | |
|
153 | 0 | return _buf; |
154 | 0 | } |
155 | | |
156 | 0 | int Errno::no() { |
157 | 0 | return errno; |
158 | 0 | } |
159 | | |
160 | | template <> |
161 | 4 | bool valid_signed_number<int128_t>(const std::string& value_str) { |
162 | 4 | char* endptr = nullptr; |
163 | 4 | const char* value_string = value_str.c_str(); |
164 | 4 | int64_t value = strtol(value_string, &endptr, 10); |
165 | 4 | if (*endptr != 0) { |
166 | 0 | return false; |
167 | 4 | } else if (value > LONG_MIN && value < LONG_MAX) { |
168 | 2 | return true; |
169 | 2 | } else { |
170 | 2 | bool sign = false; |
171 | 2 | if (*value_string == '-' || *value_string == '+') { |
172 | 1 | if (*(value_string++) == '-') { |
173 | 1 | sign = true; |
174 | 1 | } |
175 | 1 | } |
176 | | |
177 | 2 | uint128_t current = 0; |
178 | 2 | uint128_t max_int128 = std::numeric_limits<int128_t>::max(); |
179 | 80 | while (*value_string != 0) { |
180 | 78 | if (current > max_int128 / 10) { |
181 | 0 | return false; |
182 | 0 | } |
183 | | |
184 | 78 | current = current * 10 + (*(value_string++) - '0'); |
185 | 78 | } |
186 | | |
187 | 2 | if ((!sign && current > max_int128) || (sign && current > max_int128 + 1)) { |
188 | 2 | return false; |
189 | 2 | } |
190 | | |
191 | 0 | return true; |
192 | 2 | } |
193 | 4 | } |
194 | | |
195 | 7 | bool valid_decimal(const std::string& value_str, const uint32_t precision, const uint32_t frac) { |
196 | 7 | const char* decimal_pattern = "-?(\\d+)(.\\d+)?"; |
197 | 7 | std::regex e(decimal_pattern); |
198 | 7 | std::smatch what; |
199 | 7 | if (!std::regex_match(value_str, what, e) || what[0].str().size() != value_str.size()) { |
200 | 1 | LOG(WARNING) << "invalid decimal value. [value=" << value_str << "]"; |
201 | 1 | return false; |
202 | 1 | } |
203 | | |
204 | 6 | size_t number_length = value_str.size(); |
205 | 6 | bool is_negative = value_str[0] == '-'; |
206 | 6 | if (is_negative) { |
207 | 2 | --number_length; |
208 | 2 | } |
209 | | |
210 | 6 | size_t integer_len = 0; |
211 | 6 | size_t fractional_len = 0; |
212 | 6 | size_t point_pos = value_str.find('.'); |
213 | 6 | if (point_pos == std::string::npos) { |
214 | 2 | integer_len = number_length; |
215 | 2 | fractional_len = 0; |
216 | 4 | } else { |
217 | 4 | integer_len = point_pos - (is_negative ? 1 : 0); |
218 | 4 | fractional_len = number_length - point_pos - 1; |
219 | 4 | } |
220 | | |
221 | | /// For value likes "0.xxxxxx", the integer_len should actually be 0. |
222 | 6 | if (integer_len == 1 && precision - frac == 0) { |
223 | 0 | if (what[1].str() == "0") { |
224 | 0 | integer_len = 0; |
225 | 0 | } |
226 | 0 | } |
227 | | |
228 | 6 | return (integer_len <= (precision - frac) && fractional_len <= frac); |
229 | 7 | } |
230 | | |
231 | 27 | bool valid_datetime(const std::string& value_str, const uint32_t scale) { |
232 | 27 | constexpr uint32_t max_datetime_scale = 6; |
233 | 27 | constexpr uint32_t max_timestamp_scale = 9; |
234 | 27 | DORIS_CHECK_LE(scale, max_timestamp_scale); |
235 | | |
236 | 27 | const char* datetime_pattern = |
237 | 27 | "((?:\\d){4})-((?:\\d){2})-((?:\\d){2})[ ]*" |
238 | 27 | "(((?:\\d){2}):((?:\\d){2}):((?:\\d){2})([.]*((?:\\d){0,9})))?" |
239 | 27 | "[ ]*([+-](?:\\d){2}:(?:\\d){2})?"; |
240 | | |
241 | 27 | std::regex e(datetime_pattern); |
242 | 27 | std::smatch what; |
243 | | |
244 | 27 | if (std::regex_match(value_str, what, e)) { |
245 | 24 | if (what[0].str().size() != value_str.size()) { |
246 | 0 | LOG(WARNING) << "datetime str does not fully match. [value_str=" << value_str |
247 | 0 | << " match=" << what[0].str() << "]"; |
248 | 0 | return false; |
249 | 0 | } |
250 | | |
251 | 24 | int64_t month = strtol(what[2].str().c_str(), nullptr, 10); |
252 | 24 | if (month < 1 || month > 12) { |
253 | 2 | LOG(WARNING) << "invalid month. [month=" << month << "]"; |
254 | 2 | return false; |
255 | 2 | } |
256 | | |
257 | 22 | int64_t day = strtol(what[3].str().c_str(), nullptr, 10); |
258 | 22 | if (day < 1 || day > 31) { |
259 | 2 | LOG(WARNING) << "invalid day. [day=" << day << "]"; |
260 | 2 | return false; |
261 | 2 | } |
262 | | |
263 | 20 | if (what[4].length()) { |
264 | 19 | int64_t hour = strtol(what[5].str().c_str(), nullptr, 10); |
265 | 19 | if (hour < 0 || hour > 23) { |
266 | 1 | LOG(WARNING) << "invalid hour. [hour=" << hour << "]"; |
267 | 1 | return false; |
268 | 1 | } |
269 | | |
270 | 18 | int64_t minute = strtol(what[6].str().c_str(), nullptr, 10); |
271 | 18 | if (minute < 0 || minute > 59) { |
272 | 1 | LOG(WARNING) << "invalid minute. [minute=" << minute << "]"; |
273 | 1 | return false; |
274 | 1 | } |
275 | | |
276 | 17 | int64_t second = strtol(what[7].str().c_str(), nullptr, 10); |
277 | 17 | if (second < 0 || second > 59) { |
278 | 1 | LOG(WARNING) << "invalid second. [second=" << second << "]"; |
279 | 1 | return false; |
280 | 1 | } |
281 | 16 | if (what[8].length()) { |
282 | 2 | const auto fractional_second = what[9].str(); |
283 | 2 | const auto max_fractional_digits = |
284 | 2 | scale > max_datetime_scale ? scale : max_datetime_scale; |
285 | 2 | if (fractional_second.size() > max_fractional_digits) { |
286 | 1 | LOG(WARNING) << "invalid fractional second. [fractional_second=" |
287 | 1 | << fractional_second << "]"; |
288 | 1 | return false; |
289 | 1 | } |
290 | 1 | if (fractional_second.size() > scale && |
291 | 1 | fractional_second.find_first_not_of('0', scale) != std::string::npos) { |
292 | 0 | LOG(WARNING) << "invalid fractional second. [fractional_second=" |
293 | 0 | << fractional_second << ", scale = " << scale << "]"; |
294 | 0 | return false; |
295 | 0 | } |
296 | 1 | } |
297 | 16 | } |
298 | | |
299 | 16 | return true; |
300 | 20 | } else { |
301 | 3 | LOG(WARNING) << "datetime string does not match"; |
302 | 3 | return false; |
303 | 3 | } |
304 | 27 | } |
305 | | |
306 | 0 | bool valid_bool(const std::string& value_str) { |
307 | 0 | if (value_str == "0" || value_str == "1") { |
308 | 0 | return true; |
309 | 0 | } |
310 | 0 | StringParser::ParseResult result; |
311 | 0 | StringParser::string_to_bool(value_str.c_str(), value_str.length(), &result); |
312 | 0 | return result == StringParser::PARSE_SUCCESS; |
313 | 0 | } |
314 | | |
315 | 5 | bool valid_ipv4(const std::string& value_str) { |
316 | 5 | return IPv4Value::is_valid_string(value_str.c_str(), value_str.size()); |
317 | 5 | } |
318 | | |
319 | 8 | bool valid_ipv6(const std::string& value_str) { |
320 | 8 | return IPv6Value::is_valid_string(value_str.c_str(), value_str.size()); |
321 | 8 | } |
322 | | |
323 | 0 | bool valid_uuid(const std::string& value_str) { |
324 | 0 | return UUIDValue::is_valid_string(value_str.c_str(), value_str.size()); |
325 | 0 | } |
326 | | #include "common/compile_check_end.h" |
327 | | } // namespace doris |