Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/Cache/FileCache_fwd.h |
19 | | // and modified by Doris |
20 | | |
21 | | #include "io/cache/file_cache_common.h" |
22 | | |
23 | | #include "common/config.h" |
24 | | #include "exec/common/hex.h" |
25 | | #include "io/cache/block_file_cache.h" |
26 | | |
27 | | namespace doris::io { |
28 | | |
29 | 1.17k | std::string cache_type_to_surfix(FileCacheType type) { |
30 | 1.17k | switch (type) { |
31 | 200 | case FileCacheType::INDEX: |
32 | 200 | return "_idx"; |
33 | 210 | case FileCacheType::DISPOSABLE: |
34 | 210 | return "_disposable"; |
35 | 756 | case FileCacheType::NORMAL: |
36 | 756 | return ""; |
37 | 0 | case FileCacheType::TTL: |
38 | 0 | return "_ttl"; |
39 | 5 | case FileCacheType::META: |
40 | 5 | return "_meta"; |
41 | 1.17k | } |
42 | 0 | return ""; |
43 | 1.17k | } |
44 | | |
45 | 3 | FileCacheType surfix_to_cache_type(const std::string& str) { |
46 | 3 | if (str == "idx") { |
47 | 1 | return FileCacheType::INDEX; |
48 | 2 | } else if (str == "disposable") { |
49 | 1 | return FileCacheType::DISPOSABLE; |
50 | 1 | } else if (str == "ttl") { |
51 | 1 | return FileCacheType::TTL; |
52 | 1 | } else if (str == "meta") { |
53 | 0 | return FileCacheType::META; |
54 | 0 | } |
55 | 3 | DCHECK(false) << "The string is " << str; |
56 | 0 | return FileCacheType::DISPOSABLE; |
57 | 3 | } |
58 | | |
59 | 6.59k | FileCacheType string_to_cache_type(const std::string& str) { |
60 | 6.59k | if (str == "normal") { |
61 | 1.31k | return FileCacheType::NORMAL; |
62 | 5.27k | } else if (str == "index") { |
63 | 1.31k | return FileCacheType::INDEX; |
64 | 3.95k | } else if (str == "disposable") { |
65 | 1.31k | return FileCacheType::DISPOSABLE; |
66 | 2.63k | } else if (str == "ttl") { |
67 | 1.31k | return FileCacheType::TTL; |
68 | 1.31k | } else if (str == "meta") { |
69 | 1.31k | return FileCacheType::META; |
70 | 1.31k | } |
71 | 6.59k | DCHECK(false) << "The string is " << str; |
72 | 0 | return FileCacheType::NORMAL; |
73 | 6.59k | } |
74 | 8.04k | std::string cache_type_to_string(FileCacheType type) { |
75 | 8.04k | switch (type) { |
76 | 773 | case FileCacheType::INDEX: |
77 | 773 | return "index"; |
78 | 772 | case FileCacheType::DISPOSABLE: |
79 | 772 | return "disposable"; |
80 | 3.09k | case FileCacheType::NORMAL: |
81 | 3.09k | return "normal"; |
82 | 752 | case FileCacheType::TTL: |
83 | 752 | return "ttl"; |
84 | 2.65k | case FileCacheType::META: |
85 | 2.65k | return "meta"; |
86 | 8.04k | } |
87 | 8.04k | DCHECK(false) << "unknown type: " << type; |
88 | 0 | return "normal"; |
89 | 8.04k | } |
90 | | |
91 | 166 | std::string FileCacheSettings::to_string() const { |
92 | 166 | std::stringstream ss; |
93 | 166 | ss << "capacity: " << capacity << ", max_file_block_size: " << max_file_block_size |
94 | 166 | << ", max_query_cache_size: " << max_query_cache_size |
95 | 166 | << ", disposable_queue_size: " << disposable_queue_size |
96 | 166 | << ", disposable_queue_elements: " << disposable_queue_elements |
97 | 166 | << ", index_queue_size: " << index_queue_size |
98 | 166 | << ", index_queue_elements: " << index_queue_elements |
99 | 166 | << ", ttl_queue_size: " << ttl_queue_size << ", ttl_queue_elements: " << ttl_queue_elements |
100 | 166 | << ", meta_queue_size: " << meta_queue_size |
101 | 166 | << ", meta_queue_elements: " << meta_queue_elements |
102 | 166 | << ", query_queue_size: " << query_queue_size |
103 | 166 | << ", query_queue_elements: " << query_queue_elements << ", storage: " << storage; |
104 | 166 | return ss.str(); |
105 | 166 | } |
106 | | |
107 | | FileCacheSettings get_file_cache_settings(size_t capacity, size_t max_query_cache_size, |
108 | | size_t normal_percent, size_t disposable_percent, |
109 | | size_t index_percent, size_t ttl_percent, |
110 | 7 | size_t meta_percent, const std::string& storage) { |
111 | 7 | io::FileCacheSettings settings; |
112 | 7 | settings.normal_percent = normal_percent; |
113 | 7 | settings.disposable_percent = disposable_percent; |
114 | 7 | settings.index_percent = index_percent; |
115 | 7 | settings.ttl_percent = ttl_percent; |
116 | 7 | settings.meta_percent = meta_percent; |
117 | 7 | settings.max_query_cache_size = max_query_cache_size; |
118 | 7 | settings.storage = storage; |
119 | 7 | if (capacity == 0) { |
120 | 0 | return settings; |
121 | 0 | } |
122 | 7 | settings.capacity = capacity; |
123 | 7 | settings.max_file_block_size = config::file_cache_each_block_size; |
124 | 7 | size_t per_size = settings.capacity / 100; |
125 | 7 | settings.disposable_queue_size = per_size * disposable_percent; |
126 | 7 | settings.disposable_queue_elements = |
127 | 7 | std::max(settings.disposable_queue_size / settings.max_file_block_size, |
128 | 7 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
129 | | |
130 | 7 | settings.index_queue_size = per_size * index_percent; |
131 | 7 | settings.index_queue_elements = |
132 | 7 | std::max(settings.index_queue_size / settings.max_file_block_size, |
133 | 7 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
134 | | |
135 | 7 | settings.ttl_queue_size = per_size * ttl_percent; |
136 | 7 | settings.ttl_queue_elements = std::max(settings.ttl_queue_size / settings.max_file_block_size, |
137 | 7 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
138 | | |
139 | 7 | settings.meta_queue_size = per_size * meta_percent; |
140 | 7 | settings.meta_queue_elements = std::max(settings.meta_queue_size / settings.max_file_block_size, |
141 | 7 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
142 | | |
143 | 7 | settings.query_queue_size = settings.capacity - settings.disposable_queue_size - |
144 | 7 | settings.index_queue_size - settings.ttl_queue_size - |
145 | 7 | settings.meta_queue_size; |
146 | 7 | settings.query_queue_elements = |
147 | 7 | std::max(settings.query_queue_size / settings.max_file_block_size, |
148 | 7 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
149 | 7 | return settings; |
150 | 7 | } |
151 | | |
152 | 26.7k | std::string UInt128Wrapper::to_string() const { |
153 | 26.7k | return get_hex_uint_lowercase(value_); |
154 | 26.7k | } |
155 | | |
156 | | FileBlocksHolderPtr FileCacheAllocatorBuilder::allocate_cache_holder(size_t offset, size_t size, |
157 | 0 | int64_t tablet_id) const { |
158 | 0 | CacheContext ctx; |
159 | 0 | ctx.cache_type = _expiration_time == 0 ? FileCacheType::NORMAL : FileCacheType::TTL; |
160 | 0 | ctx.expiration_time = _expiration_time; |
161 | 0 | ctx.is_cold_data = _is_cold_data; |
162 | 0 | ctx.tablet_id = tablet_id; |
163 | 0 | ReadStatistics stats; |
164 | 0 | ctx.stats = &stats; |
165 | 0 | auto holder = _cache->get_or_set(_cache_hash, offset, size, ctx); |
166 | 0 | return std::make_unique<FileBlocksHolder>(std::move(holder)); |
167 | 0 | } |
168 | | |
169 | | template size_t LRUQueue::get_capacity(std::lock_guard<std::mutex>& cache_lock) const; |
170 | | template void LRUQueue::remove(Iterator queue_it, std::lock_guard<std::mutex>& cache_lock); |
171 | | |
172 | 0 | std::string FileCacheInfo::to_string() const { |
173 | 0 | std::stringstream ss; |
174 | 0 | ss << "Hash: " << hash.to_string() << "\n" |
175 | 0 | << "Expiration Time: " << expiration_time << "\n" |
176 | 0 | << "Offset: " << offset << "\n" |
177 | 0 | << "Cache Type: " << cache_type_to_string(cache_type) << "\n"; |
178 | 0 | return ss.str(); |
179 | 0 | } |
180 | | |
181 | 0 | std::string InconsistencyType::to_string() const { |
182 | 0 | std::string result = "Inconsistency Reason: "; |
183 | 0 | if (type == NONE) { |
184 | 0 | result += "NONE"; |
185 | 0 | } else { |
186 | 0 | if (type & NOT_LOADED) { |
187 | 0 | result += "NOT_LOADED "; |
188 | 0 | } |
189 | 0 | if (type & MISSING_IN_STORAGE) { |
190 | 0 | result += "MISSING_IN_STORAGE "; |
191 | 0 | } |
192 | 0 | if (type & SIZE_INCONSISTENT) { |
193 | 0 | result += "SIZE_INCONSISTENT "; |
194 | 0 | } |
195 | 0 | if (type & CACHE_TYPE_INCONSISTENT) { |
196 | 0 | result += "CACHE_TYPE_INCONSISTENT "; |
197 | 0 | } |
198 | 0 | if (type & EXPIRATION_TIME_INCONSISTENT) { |
199 | 0 | result += "EXPIRATION_TIME_INCONSISTENT "; |
200 | 0 | } |
201 | 0 | if (type & TMP_FILE_EXPECT_DOWNLOADING_STATE) { |
202 | 0 | result += "TMP_FILE_EXPECT_DOWNLOADING_STATE"; |
203 | 0 | } |
204 | 0 | } |
205 | 0 | result += "\n"; |
206 | 0 | return result; |
207 | 0 | } |
208 | | |
209 | 1 | std::optional<int64_t> get_tablet_id(std::string file_path) { |
210 | | // Expected path formats: |
211 | | // support both .dat and .idx file extensions |
212 | | // support formate see ut. storage_resource_test:StorageResourceTest.ParseTabletIdFromPath |
213 | | |
214 | 1 | if (file_path.empty()) { |
215 | 0 | return std::nullopt; |
216 | 0 | } |
217 | | |
218 | | // Find the position of "data/" in the path |
219 | 1 | std::string_view path_view = file_path; |
220 | 1 | std::string_view data_prefix = DATA_PREFIX; |
221 | 1 | size_t data_pos = path_view.find(data_prefix); |
222 | 1 | if (data_pos == std::string_view::npos) { |
223 | 0 | return std::nullopt; |
224 | 0 | } |
225 | | |
226 | 1 | if (data_prefix.length() + data_pos >= path_view.length()) { |
227 | 1 | return std::nullopt; |
228 | 1 | } |
229 | | |
230 | | // Extract the part after "data/" |
231 | 0 | path_view = path_view.substr(data_pos + data_prefix.length() + 1); |
232 | | |
233 | | // Check if path ends with .dat or .idx |
234 | 0 | if (!path_view.ends_with(".dat") && !path_view.ends_with(".idx")) { |
235 | 0 | return std::nullopt; |
236 | 0 | } |
237 | | |
238 | | // Count slashes in the remaining path |
239 | 0 | size_t slash_count = 0; |
240 | 0 | for (char c : path_view) { |
241 | 0 | if (c == '/') { |
242 | 0 | slash_count++; |
243 | 0 | } |
244 | 0 | } |
245 | | |
246 | | // Split path by '/' |
247 | 0 | std::vector<std::string_view> parts; |
248 | 0 | size_t start = 0; |
249 | 0 | size_t pos = 0; |
250 | 0 | while ((pos = path_view.find('/', start)) != std::string_view::npos) { |
251 | 0 | if (pos > start) { |
252 | 0 | parts.push_back(path_view.substr(start, pos - start)); |
253 | 0 | } |
254 | 0 | start = pos + 1; |
255 | 0 | } |
256 | 0 | if (start < path_view.length()) { |
257 | 0 | parts.push_back(path_view.substr(start)); |
258 | 0 | } |
259 | |
|
260 | 0 | if (parts.empty()) { |
261 | 0 | return std::nullopt; |
262 | 0 | } |
263 | | |
264 | | // Determine path version based on slash count and extract tablet_id |
265 | | // Version 0: {tablet_id}/{rowset_id}_{seg_id}.dat (1 slash) |
266 | | // Version 1: {shard}/{tablet_id}/{rowset_id}/{seg_id}.dat (3 slashes) |
267 | | |
268 | 0 | if (slash_count == 1) { |
269 | | // Version 0 format: parts[0] should be tablet_id |
270 | 0 | if (parts.size() >= 1) { |
271 | 0 | try { |
272 | 0 | int64_t tablet_id = std::stoll(std::string(parts[0])); |
273 | 0 | return tablet_id; |
274 | 0 | } catch (const std::exception&) { |
275 | | // Not a valid number, return nullopt at last |
276 | 0 | } |
277 | 0 | } |
278 | 0 | } else if (slash_count == 3) { |
279 | | // Version 1 format: parts[1] should be tablet_id (parts[0] is shard) |
280 | 0 | if (parts.size() >= 2) { |
281 | 0 | try { |
282 | 0 | int64_t tablet_id = std::stoll(std::string(parts[1])); |
283 | 0 | return tablet_id; |
284 | 0 | } catch (const std::exception&) { |
285 | | // Not a valid number, return nullopt at last |
286 | 0 | } |
287 | 0 | } |
288 | 0 | } |
289 | | |
290 | 0 | return std::nullopt; |
291 | 0 | } |
292 | | |
293 | | } // namespace doris::io |