be/src/io/cache/file_cache_common.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/Cache/FileCache_fwd.h |
19 | | // and modified by Doris |
20 | | |
21 | | #include "io/cache/file_cache_common.h" |
22 | | |
23 | | #include "common/config.h" |
24 | | #include "exec/common/hex.h" |
25 | | #include "io/cache/block_file_cache.h" |
26 | | |
27 | | namespace doris::io { |
28 | | |
29 | 120k | std::string cache_type_to_surfix(FileCacheType type) { |
30 | 120k | switch (type) { |
31 | 41.9k | case FileCacheType::INDEX: |
32 | 41.9k | return "_idx"; |
33 | 4.35k | case FileCacheType::DISPOSABLE: |
34 | 4.35k | return "_disposable"; |
35 | 74.4k | case FileCacheType::NORMAL: |
36 | 74.4k | return ""; |
37 | 0 | case FileCacheType::TTL: |
38 | 0 | return "_ttl"; |
39 | 120k | } |
40 | 0 | return ""; |
41 | 120k | } |
42 | | |
43 | 3 | FileCacheType surfix_to_cache_type(const std::string& str) { |
44 | 3 | if (str == "idx") { |
45 | 1 | return FileCacheType::INDEX; |
46 | 2 | } else if (str == "disposable") { |
47 | 1 | return FileCacheType::DISPOSABLE; |
48 | 1 | } else if (str == "ttl") { |
49 | 1 | return FileCacheType::TTL; |
50 | 1 | } |
51 | 3 | DCHECK(false) << "The string is " << str; |
52 | 0 | return FileCacheType::DISPOSABLE; |
53 | 3 | } |
54 | | |
55 | 6.88k | FileCacheType string_to_cache_type(const std::string& str) { |
56 | 6.88k | if (str == "normal") { |
57 | 1.72k | return FileCacheType::NORMAL; |
58 | 5.16k | } else if (str == "index") { |
59 | 1.72k | return FileCacheType::INDEX; |
60 | 3.44k | } else if (str == "disposable") { |
61 | 1.72k | return FileCacheType::DISPOSABLE; |
62 | 1.72k | } else if (str == "ttl") { |
63 | 1.72k | return FileCacheType::TTL; |
64 | 1.72k | } |
65 | 18.4E | DCHECK(false) << "The string is " << str; |
66 | 18.4E | return FileCacheType::NORMAL; |
67 | 6.88k | } |
68 | 11.4k | std::string cache_type_to_string(FileCacheType type) { |
69 | 11.4k | switch (type) { |
70 | 2.77k | case FileCacheType::INDEX: |
71 | 2.77k | return "index"; |
72 | 232 | case FileCacheType::DISPOSABLE: |
73 | 232 | return "disposable"; |
74 | 6.90k | case FileCacheType::NORMAL: |
75 | 6.90k | return "normal"; |
76 | 1.51k | case FileCacheType::TTL: |
77 | 1.51k | return "ttl"; |
78 | 11.4k | } |
79 | 11.4k | DCHECK(false) << "unknown type: " << type; |
80 | 0 | return "normal"; |
81 | 11.4k | } |
82 | | |
83 | 164 | std::string FileCacheSettings::to_string() const { |
84 | 164 | std::stringstream ss; |
85 | 164 | ss << "capacity: " << capacity << ", max_file_block_size: " << max_file_block_size |
86 | 164 | << ", max_query_cache_size: " << max_query_cache_size |
87 | 164 | << ", disposable_queue_size: " << disposable_queue_size |
88 | 164 | << ", disposable_queue_elements: " << disposable_queue_elements |
89 | 164 | << ", index_queue_size: " << index_queue_size |
90 | 164 | << ", index_queue_elements: " << index_queue_elements |
91 | 164 | << ", ttl_queue_size: " << ttl_queue_size << ", ttl_queue_elements: " << ttl_queue_elements |
92 | 164 | << ", query_queue_size: " << query_queue_size |
93 | 164 | << ", query_queue_elements: " << query_queue_elements << ", storage: " << storage; |
94 | 164 | return ss.str(); |
95 | 164 | } |
96 | | |
97 | | FileCacheSettings get_file_cache_settings(size_t capacity, size_t max_query_cache_size, |
98 | | size_t normal_percent, size_t disposable_percent, |
99 | | size_t index_percent, size_t ttl_percent, |
100 | 9 | const std::string& storage) { |
101 | 9 | io::FileCacheSettings settings; |
102 | 9 | if (capacity == 0) { |
103 | 0 | return settings; |
104 | 0 | } |
105 | 9 | settings.capacity = capacity; |
106 | 9 | settings.max_file_block_size = config::file_cache_each_block_size; |
107 | 9 | settings.max_query_cache_size = max_query_cache_size; |
108 | 9 | size_t per_size = settings.capacity / 100; |
109 | 9 | settings.disposable_queue_size = per_size * disposable_percent; |
110 | 9 | settings.disposable_queue_elements = |
111 | 9 | std::max(settings.disposable_queue_size / settings.max_file_block_size, |
112 | 9 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
113 | | |
114 | 9 | settings.index_queue_size = per_size * index_percent; |
115 | 9 | settings.index_queue_elements = |
116 | 9 | std::max(settings.index_queue_size / settings.max_file_block_size, |
117 | 9 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
118 | | |
119 | 9 | settings.ttl_queue_size = per_size * ttl_percent; |
120 | 9 | settings.ttl_queue_elements = std::max(settings.ttl_queue_size / settings.max_file_block_size, |
121 | 9 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
122 | | |
123 | 9 | settings.query_queue_size = settings.capacity - settings.disposable_queue_size - |
124 | 9 | settings.index_queue_size - settings.ttl_queue_size; |
125 | 9 | settings.query_queue_elements = |
126 | 9 | std::max(settings.query_queue_size / settings.max_file_block_size, |
127 | 9 | REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS); |
128 | 9 | settings.storage = storage; |
129 | 9 | return settings; |
130 | 9 | } |
131 | | |
132 | 591k | std::string UInt128Wrapper::to_string() const { |
133 | 591k | return get_hex_uint_lowercase(value_); |
134 | 591k | } |
135 | | |
136 | | FileBlocksHolderPtr FileCacheAllocatorBuilder::allocate_cache_holder(size_t offset, size_t size, |
137 | 61.2k | int64_t tablet_id) const { |
138 | 61.2k | CacheContext ctx; |
139 | 18.4E | ctx.cache_type = _expiration_time == 0 ? FileCacheType::NORMAL : FileCacheType::TTL; |
140 | 61.2k | ctx.expiration_time = _expiration_time; |
141 | 61.2k | ctx.is_cold_data = _is_cold_data; |
142 | 61.2k | ctx.tablet_id = tablet_id; |
143 | 61.2k | ReadStatistics stats; |
144 | 61.2k | ctx.stats = &stats; |
145 | 61.2k | auto holder = _cache->get_or_set(_cache_hash, offset, size, ctx); |
146 | 61.2k | return std::make_unique<FileBlocksHolder>(std::move(holder)); |
147 | 61.2k | } |
148 | | |
149 | | template size_t LRUQueue::get_capacity(std::lock_guard<std::mutex>& cache_lock) const; |
150 | | template void LRUQueue::remove(Iterator queue_it, std::lock_guard<std::mutex>& cache_lock); |
151 | | |
152 | 0 | std::string FileCacheInfo::to_string() const { |
153 | 0 | std::stringstream ss; |
154 | 0 | ss << "Hash: " << hash.to_string() << "\n" |
155 | 0 | << "Expiration Time: " << expiration_time << "\n" |
156 | 0 | << "Offset: " << offset << "\n" |
157 | 0 | << "Cache Type: " << cache_type_to_string(cache_type) << "\n"; |
158 | 0 | return ss.str(); |
159 | 0 | } |
160 | | |
161 | 0 | std::string InconsistencyType::to_string() const { |
162 | 0 | std::string result = "Inconsistency Reason: "; |
163 | 0 | if (type == NONE) { |
164 | 0 | result += "NONE"; |
165 | 0 | } else { |
166 | 0 | if (type & NOT_LOADED) { |
167 | 0 | result += "NOT_LOADED "; |
168 | 0 | } |
169 | 0 | if (type & MISSING_IN_STORAGE) { |
170 | 0 | result += "MISSING_IN_STORAGE "; |
171 | 0 | } |
172 | 0 | if (type & SIZE_INCONSISTENT) { |
173 | 0 | result += "SIZE_INCONSISTENT "; |
174 | 0 | } |
175 | 0 | if (type & CACHE_TYPE_INCONSISTENT) { |
176 | 0 | result += "CACHE_TYPE_INCONSISTENT "; |
177 | 0 | } |
178 | 0 | if (type & EXPIRATION_TIME_INCONSISTENT) { |
179 | 0 | result += "EXPIRATION_TIME_INCONSISTENT "; |
180 | 0 | } |
181 | 0 | if (type & TMP_FILE_EXPECT_DOWNLOADING_STATE) { |
182 | 0 | result += "TMP_FILE_EXPECT_DOWNLOADING_STATE"; |
183 | 0 | } |
184 | 0 | } |
185 | 0 | result += "\n"; |
186 | 0 | return result; |
187 | 0 | } |
188 | | |
189 | 6.42M | std::optional<int64_t> get_tablet_id(std::string file_path) { |
190 | | // Expected path formats: |
191 | | // support both .dat and .idx file extensions |
192 | | // support formate see ut. storage_resource_test:StorageResourceTest.ParseTabletIdFromPath |
193 | | |
194 | 6.42M | if (file_path.empty()) { |
195 | 0 | return std::nullopt; |
196 | 0 | } |
197 | | |
198 | | // Find the position of "data/" in the path |
199 | 6.42M | std::string_view path_view = file_path; |
200 | 6.42M | std::string_view data_prefix = DATA_PREFIX; |
201 | 6.42M | size_t data_pos = path_view.find(data_prefix); |
202 | 6.42M | if (data_pos == std::string_view::npos) { |
203 | 557 | return std::nullopt; |
204 | 557 | } |
205 | | |
206 | 6.42M | if (data_prefix.length() + data_pos >= path_view.length()) { |
207 | 1 | return std::nullopt; |
208 | 1 | } |
209 | | |
210 | | // Extract the part after "data/" |
211 | 6.42M | path_view = path_view.substr(data_pos + data_prefix.length() + 1); |
212 | | |
213 | | // Check if path ends with .dat or .idx |
214 | 6.42M | if (!path_view.ends_with(".dat") && !path_view.ends_with(".idx")) { |
215 | 530 | return std::nullopt; |
216 | 530 | } |
217 | | |
218 | | // Count slashes in the remaining path |
219 | 6.42M | size_t slash_count = 0; |
220 | 436M | for (char c : path_view) { |
221 | 436M | if (c == '/') { |
222 | 6.43M | slash_count++; |
223 | 6.43M | } |
224 | 436M | } |
225 | | |
226 | | // Split path by '/' |
227 | 6.42M | std::vector<std::string_view> parts; |
228 | 6.42M | size_t start = 0; |
229 | 6.42M | size_t pos = 0; |
230 | 12.8M | while ((pos = path_view.find('/', start)) != std::string_view::npos) { |
231 | 6.43M | if (pos > start) { |
232 | 6.42M | parts.push_back(path_view.substr(start, pos - start)); |
233 | 6.42M | } |
234 | 6.43M | start = pos + 1; |
235 | 6.43M | } |
236 | 6.43M | if (start < path_view.length()) { |
237 | 6.43M | parts.push_back(path_view.substr(start)); |
238 | 6.43M | } |
239 | | |
240 | 6.42M | if (parts.empty()) { |
241 | 0 | return std::nullopt; |
242 | 0 | } |
243 | | |
244 | | // Determine path version based on slash count and extract tablet_id |
245 | | // Version 0: {tablet_id}/{rowset_id}_{seg_id}.dat (1 slash) |
246 | | // Version 1: {shard}/{tablet_id}/{rowset_id}/{seg_id}.dat (3 slashes) |
247 | | |
248 | 6.42M | if (slash_count == 1) { |
249 | | // Version 0 format: parts[0] should be tablet_id |
250 | 6.43M | if (parts.size() >= 1) { |
251 | 6.43M | try { |
252 | 6.43M | int64_t tablet_id = std::stoll(std::string(parts[0])); |
253 | 6.43M | return tablet_id; |
254 | 6.43M | } catch (const std::exception&) { |
255 | | // Not a valid number, return nullopt at last |
256 | 0 | } |
257 | 6.43M | } |
258 | 18.4E | } else if (slash_count == 3) { |
259 | | // Version 1 format: parts[1] should be tablet_id (parts[0] is shard) |
260 | 0 | if (parts.size() >= 2) { |
261 | 0 | try { |
262 | 0 | int64_t tablet_id = std::stoll(std::string(parts[1])); |
263 | 0 | return tablet_id; |
264 | 0 | } catch (const std::exception&) { |
265 | | // Not a valid number, return nullopt at last |
266 | 0 | } |
267 | 0 | } |
268 | 0 | } |
269 | | |
270 | 18.4E | return std::nullopt; |
271 | 6.42M | } |
272 | | |
273 | | } // namespace doris::io |