/root/doris/be/src/util/timezone_utils.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "util/timezone_utils.h" |
19 | | |
20 | | #include <cctz/civil_time.h> |
21 | | #include <cctz/time_zone.h> |
22 | | #include <fcntl.h> |
23 | | #include <glog/logging.h> |
24 | | #include <re2/re2.h> |
25 | | #include <re2/stringpiece.h> |
26 | | #include <sys/mman.h> |
27 | | #include <sys/stat.h> |
28 | | #include <sys/types.h> |
29 | | #include <unistd.h> |
30 | | |
31 | | #include <algorithm> |
32 | | #include <boost/algorithm/string.hpp> |
33 | | #include <boost/algorithm/string/case_conv.hpp> |
34 | | #include <cctype> |
35 | | #include <chrono> |
36 | | #include <cstdlib> |
37 | | #include <filesystem> |
38 | | #include <memory> |
39 | | #include <string> |
40 | | #include <string_view> |
41 | | |
42 | | #include "common/exception.h" |
43 | | #include "common/logging.h" |
44 | | #include "common/status.h" |
45 | | |
46 | | using boost::algorithm::to_lower_copy; |
47 | | |
48 | | namespace fs = std::filesystem; |
49 | | #include "common/compile_check_begin.h" |
50 | | |
51 | | namespace doris { |
52 | | |
53 | | using ZoneList = std::unordered_map<std::string, cctz::time_zone>; |
54 | | |
55 | | RE2 time_zone_offset_format_reg(R"(^[+-]{1}\d{2}\:\d{2}$)"); // visiting is thread-safe |
56 | | |
57 | | // for ut, make it never nullptr. |
58 | | std::unique_ptr<ZoneList> lower_zone_cache_ = std::make_unique<ZoneList>(); |
59 | | |
60 | | const std::string TimezoneUtils::default_time_zone = "+08:00"; |
61 | | static const char* tzdir = "/usr/share/zoneinfo"; // default value, may change by TZDIR env var |
62 | | |
63 | 8 | void TimezoneUtils::clear_timezone_caches() { |
64 | 8 | lower_zone_cache_->clear(); |
65 | 8 | } |
66 | 2 | size_t TimezoneUtils::cache_size() { |
67 | 2 | return lower_zone_cache_->size(); |
68 | 2 | } |
69 | | |
70 | 155k | static bool parse_save_name_tz(const std::string& tz_name) { |
71 | 155k | cctz::time_zone tz; |
72 | 155k | PROPAGATE_FALSE(cctz::load_time_zone(tz_name, &tz)); |
73 | 154k | lower_zone_cache_->emplace(to_lower_copy(tz_name), tz); |
74 | 154k | return true; |
75 | 155k | } |
76 | | |
77 | 260 | void TimezoneUtils::load_timezones_to_cache() { |
78 | 260 | std::string base_str; |
79 | | // try get from system |
80 | 260 | char* tzdir_env = std::getenv("TZDIR"); |
81 | 260 | if (tzdir_env && *tzdir_env) { |
82 | 0 | tzdir = tzdir_env; |
83 | 0 | } |
84 | | |
85 | 260 | base_str = tzdir; |
86 | 260 | base_str += '/'; |
87 | | |
88 | 260 | const auto root_path = fs::path {base_str}; |
89 | 260 | if (!exists(root_path)) { |
90 | 0 | throw Exception(Status::FatalError("Cannot find system tzfile. Doris exiting!")); |
91 | 0 | } |
92 | | |
93 | 260 | std::set<std::string> ignore_paths = {"posix", "right"}; // duplications. ignore them. |
94 | | |
95 | 161k | for (fs::recursive_directory_iterator it {base_str}; it != end(it); it++) { |
96 | 161k | const auto& dir_entry = *it; |
97 | 161k | if (dir_entry.is_regular_file() || |
98 | 161k | (dir_entry.is_symlink() && is_regular_file(read_symlink(dir_entry)))) { |
99 | 155k | auto tz_name = dir_entry.path().string().substr(base_str.length()); |
100 | 155k | if (!parse_save_name_tz(tz_name)) { |
101 | 1.30k | LOG(WARNING) << "Meet illegal tzdata file: " << tz_name << ". skipped"; |
102 | 1.30k | } |
103 | 155k | } else if (dir_entry.is_directory() && ignore_paths.contains(dir_entry.path().filename())) { |
104 | 520 | it.disable_recursion_pending(); |
105 | 520 | } |
106 | 161k | } |
107 | | // some special cases. Z = Zulu. CST = Asia/Shanghai |
108 | 260 | if (auto it = lower_zone_cache_->find("zulu"); it != lower_zone_cache_->end()) { |
109 | 260 | lower_zone_cache_->emplace("z", it->second); |
110 | 260 | } |
111 | 260 | if (auto it = lower_zone_cache_->find("asia/shanghai"); it != lower_zone_cache_->end()) { |
112 | 260 | lower_zone_cache_->emplace("cst", it->second); |
113 | 260 | } |
114 | | |
115 | 260 | lower_zone_cache_->erase("lmt"); // local mean time for every timezone |
116 | | |
117 | 260 | load_offsets_to_cache(); |
118 | 260 | LOG(INFO) << "Preloaded" << lower_zone_cache_->size() << " timezones."; |
119 | 260 | } |
120 | | |
121 | 22.8k | static std::string to_hour_string(int arg) { |
122 | 22.8k | if (arg < 0 && arg > -10) { // -9 to -1 |
123 | 7.61k | return std::string {"-0"} + std::to_string(std::abs(arg)); |
124 | 15.2k | } else if (arg >= 0 && arg < 10) { //0 to 9 |
125 | 8.46k | return std::string {"0"} + std::to_string(arg); |
126 | 8.46k | } |
127 | 6.76k | return std::to_string(arg); |
128 | 22.8k | } |
129 | | |
130 | 282 | void TimezoneUtils::load_offsets_to_cache() { |
131 | 282 | static constexpr int supported_minutes[] = {0, 30, 45}; |
132 | 7.89k | for (int hour = -12; hour <= +14; hour++) { |
133 | 22.8k | for (int minute : supported_minutes) { |
134 | 22.8k | char min_str[3]; |
135 | 22.8k | snprintf(min_str, sizeof(min_str), "%02d", minute); |
136 | 22.8k | std::string offset_str = (hour >= 0 ? "+" : "") + to_hour_string(hour) + ':' + min_str; |
137 | 22.8k | cctz::time_zone result; |
138 | | // Rejected endpoint minutes must not be cached as the default UTC zone. |
139 | 22.8k | if (!parse_tz_offset_string(offset_str, result)) { |
140 | 1.12k | continue; |
141 | 1.12k | } |
142 | 21.7k | lower_zone_cache_->emplace(offset_str, result); |
143 | 21.7k | } |
144 | 7.61k | } |
145 | | // -00 for hour is also valid |
146 | 282 | std::string offset_str = "-00:00"; |
147 | 282 | cctz::time_zone result; |
148 | 282 | parse_tz_offset_string(offset_str, result); |
149 | 282 | lower_zone_cache_->emplace(offset_str, result); |
150 | 282 | offset_str = "-00:30"; |
151 | 282 | parse_tz_offset_string(offset_str, result); |
152 | 282 | lower_zone_cache_->emplace(offset_str, result); |
153 | 282 | offset_str = "-00:45"; |
154 | 282 | parse_tz_offset_string(offset_str, result); |
155 | 282 | lower_zone_cache_->emplace(offset_str, result); |
156 | 282 | } |
157 | | |
158 | 184k | bool TimezoneUtils::find_cctz_time_zone(const std::string& timezone, cctz::time_zone& ctz) { |
159 | 184k | if (auto it = lower_zone_cache_->find(to_lower_copy(timezone)); it != lower_zone_cache_->end()) |
160 | 135k | [[likely]] { |
161 | 135k | ctz = it->second; |
162 | 135k | return true; |
163 | 135k | } |
164 | | |
165 | 48.8k | std::string normalized; |
166 | 48.8k | if (!normalize_timezone_name(timezone, &normalized)) { |
167 | 60 | return false; |
168 | 60 | } |
169 | 48.8k | if (auto it = lower_zone_cache_->find(to_lower_copy(normalized)); |
170 | 48.8k | it != lower_zone_cache_->end()) [[likely]] { |
171 | 10 | ctz = it->second; |
172 | 10 | return true; |
173 | 10 | } |
174 | 48.7k | return parse_tz_offset_string(normalized, ctz); |
175 | 48.8k | } |
176 | | |
177 | | bool TimezoneUtils::try_get_fixed_offset_seconds(const cctz::time_zone& timezone, |
178 | 27 | int32_t* offset_seconds) { |
179 | 27 | DORIS_CHECK(offset_seconds != nullptr); |
180 | 27 | const std::string& timezone_name = timezone.name(); |
181 | 27 | if (timezone_name == "UTC" || timezone_name == "Etc/UTC" || timezone_name == "Etc/GMT") { |
182 | 15 | *offset_seconds = 0; |
183 | 15 | return true; |
184 | 15 | } |
185 | | |
186 | | // cctz names fixed_time_zone() instances with the "Fixed/" prefix. TZDB's Etc/GMT* |
187 | | // zones are fixed offsets too; cctz handles their POSIX-style reversed sign in lookup_offset(). |
188 | | // If this naming convention changes, falling through to the generic path remains correct. |
189 | 12 | static const auto epoch = std::chrono::time_point_cast<cctz::sys_seconds>( |
190 | 12 | std::chrono::system_clock::from_time_t(0)); |
191 | 12 | if (timezone_name.compare(0, 6, "Fixed/") == 0 || timezone_name.compare(0, 7, "Etc/GMT") == 0) { |
192 | 4 | *offset_seconds = timezone.lookup_offset(epoch).offset; |
193 | 4 | return true; |
194 | 4 | } |
195 | 8 | return false; |
196 | 12 | } |
197 | | |
198 | | static bool normalize_offset_string(const std::string& timezone, bool allow_hour_only, |
199 | 121k | std::string* normalized) { |
200 | 121k | if (timezone.size() < 2 || (timezone[0] != '+' && timezone[0] != '-')) { |
201 | 0 | return false; |
202 | 0 | } |
203 | | |
204 | 121k | const bool positive = timezone[0] == '+'; |
205 | 121k | const std::string_view rest(timezone.data() + 1, timezone.size() - 1); |
206 | 121k | int hour = 0; |
207 | 121k | int minute = 0; |
208 | | |
209 | 242k | const auto parse_digit = [](char c) -> int { return c - '0'; }; |
210 | 121k | const auto is_two_digits = [](std::string_view value) -> bool { |
211 | 121k | return value.size() == 2 && std::isdigit(static_cast<unsigned char>(value[0])) && |
212 | 121k | std::isdigit(static_cast<unsigned char>(value[1])); |
213 | 121k | }; |
214 | 121k | const auto is_one_or_two_digits = [](std::string_view value) -> bool { |
215 | 121k | return (value.size() == 1 || value.size() == 2) && |
216 | 121k | std::all_of(value.begin(), value.end(), |
217 | 242k | [](char c) { return std::isdigit(static_cast<unsigned char>(c)); }); |
218 | 121k | }; |
219 | | |
220 | 121k | const auto colon_pos = rest.find(':'); |
221 | 121k | if (colon_pos != std::string_view::npos) { |
222 | 121k | const std::string_view hour_part = rest.substr(0, colon_pos); |
223 | 121k | const std::string_view minute_part = rest.substr(colon_pos + 1); |
224 | 121k | if (!is_one_or_two_digits(hour_part) || !is_two_digits(minute_part)) { |
225 | 0 | return false; |
226 | 0 | } |
227 | 121k | hour = std::stoi(std::string(hour_part)); |
228 | 121k | minute = parse_digit(minute_part[0]) * 10 + parse_digit(minute_part[1]); |
229 | 121k | } else { |
230 | 8 | if (!allow_hour_only || !is_one_or_two_digits(rest)) { |
231 | 2 | return false; |
232 | 2 | } |
233 | 6 | hour = std::stoi(std::string(rest)); |
234 | 6 | } |
235 | | |
236 | | // Session fixed offsets include the endpoints only when their minutes are zero. |
237 | 121k | if (minute >= 60 || hour * 60 + minute > (positive ? 14 : 12) * 60) { |
238 | 1.15k | return false; |
239 | 1.15k | } |
240 | | |
241 | 119k | *normalized = std::string(1, positive ? '+' : '-') + (hour < 10 ? "0" : "") + |
242 | 119k | std::to_string(hour) + ":" + (minute < 10 ? "0" : "") + std::to_string(minute); |
243 | 119k | return true; |
244 | 121k | } |
245 | | |
246 | 121k | bool TimezoneUtils::normalize_timezone_name(const std::string& timezone, std::string* normalized) { |
247 | 121k | DORIS_CHECK(normalized != nullptr); |
248 | 121k | const std::string lower = to_lower_copy(timezone); |
249 | 121k | if (lower == "utc" || lower == "etc/utc" || lower == "zulu") { |
250 | 294 | *normalized = "UTC"; |
251 | 294 | return true; |
252 | 294 | } |
253 | | |
254 | 121k | if (lower.rfind("utc", 0) == 0 || lower.rfind("gmt", 0) == 0) { |
255 | 40 | if (timezone.size() <= 3) { |
256 | 0 | return false; |
257 | 0 | } |
258 | 40 | return normalize_offset_string(timezone.substr(3), true, normalized); |
259 | 40 | } |
260 | | |
261 | 121k | if (!timezone.empty() && (timezone[0] == '+' || timezone[0] == '-')) { |
262 | 121k | return normalize_offset_string(timezone, false, normalized); |
263 | 121k | } |
264 | | |
265 | 44 | return false; |
266 | 121k | } |
267 | | |
268 | 72.5k | bool TimezoneUtils::parse_tz_offset_string(const std::string& timezone, cctz::time_zone& ctz) { |
269 | 72.5k | std::string normalized; |
270 | 72.5k | if (!normalize_timezone_name(timezone, &normalized)) { |
271 | 1.14k | return false; |
272 | 1.14k | } |
273 | 71.3k | if (normalized == "UTC") { |
274 | 147 | ctz = cctz::utc_time_zone(); |
275 | 147 | return true; |
276 | 147 | } |
277 | | |
278 | 71.2k | re2::StringPiece value; |
279 | 71.2k | if (time_zone_offset_format_reg.Match(normalized, 0, normalized.size(), RE2::UNANCHORED, &value, |
280 | 71.2k | 1)) [[likely]] { |
281 | 71.2k | const bool positive = value[0] != '-'; |
282 | 71.2k | const int hour = std::stoi(value.substr(1, 2).as_string()); |
283 | 71.2k | const int minute = std::stoi(value.substr(4, 2).as_string()); |
284 | 71.2k | int offset = hour * 60 * 60 + minute * 60; |
285 | 71.2k | offset *= positive ? 1 : -1; |
286 | 71.2k | ctz = cctz::fixed_time_zone(cctz::seconds(offset)); |
287 | 71.2k | return true; |
288 | 71.2k | } |
289 | 0 | return false; |
290 | 71.2k | } |
291 | | |
292 | | #include "common/compile_check_end.h" |
293 | | } // namespace doris |