Coverage Report

Created: 2026-03-12 16:03

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/cache/file_cache_common.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
// This file is copied from
18
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/Cache/FileCache_fwd.h
19
// and modified by Doris
20
21
#include "io/cache/file_cache_common.h"
22
23
#include "common/config.h"
24
#include "exec/common/hex.h"
25
#include "io/cache/block_file_cache.h"
26
27
namespace doris::io {
28
29
120k
std::string cache_type_to_surfix(FileCacheType type) {
30
120k
    switch (type) {
31
41.9k
    case FileCacheType::INDEX:
32
41.9k
        return "_idx";
33
4.35k
    case FileCacheType::DISPOSABLE:
34
4.35k
        return "_disposable";
35
74.4k
    case FileCacheType::NORMAL:
36
74.4k
        return "";
37
0
    case FileCacheType::TTL:
38
0
        return "_ttl";
39
120k
    }
40
0
    return "";
41
120k
}
42
43
3
FileCacheType surfix_to_cache_type(const std::string& str) {
44
3
    if (str == "idx") {
45
1
        return FileCacheType::INDEX;
46
2
    } else if (str == "disposable") {
47
1
        return FileCacheType::DISPOSABLE;
48
1
    } else if (str == "ttl") {
49
1
        return FileCacheType::TTL;
50
1
    }
51
3
    DCHECK(false) << "The string is " << str;
52
0
    return FileCacheType::DISPOSABLE;
53
3
}
54
55
6.88k
FileCacheType string_to_cache_type(const std::string& str) {
56
6.88k
    if (str == "normal") {
57
1.72k
        return FileCacheType::NORMAL;
58
5.16k
    } else if (str == "index") {
59
1.72k
        return FileCacheType::INDEX;
60
3.44k
    } else if (str == "disposable") {
61
1.72k
        return FileCacheType::DISPOSABLE;
62
1.72k
    } else if (str == "ttl") {
63
1.72k
        return FileCacheType::TTL;
64
1.72k
    }
65
18.4E
    DCHECK(false) << "The string is " << str;
66
18.4E
    return FileCacheType::NORMAL;
67
6.88k
}
68
11.4k
std::string cache_type_to_string(FileCacheType type) {
69
11.4k
    switch (type) {
70
2.77k
    case FileCacheType::INDEX:
71
2.77k
        return "index";
72
232
    case FileCacheType::DISPOSABLE:
73
232
        return "disposable";
74
6.90k
    case FileCacheType::NORMAL:
75
6.90k
        return "normal";
76
1.51k
    case FileCacheType::TTL:
77
1.51k
        return "ttl";
78
11.4k
    }
79
11.4k
    DCHECK(false) << "unknown type: " << type;
80
0
    return "normal";
81
11.4k
}
82
83
164
std::string FileCacheSettings::to_string() const {
84
164
    std::stringstream ss;
85
164
    ss << "capacity: " << capacity << ", max_file_block_size: " << max_file_block_size
86
164
       << ", max_query_cache_size: " << max_query_cache_size
87
164
       << ", disposable_queue_size: " << disposable_queue_size
88
164
       << ", disposable_queue_elements: " << disposable_queue_elements
89
164
       << ", index_queue_size: " << index_queue_size
90
164
       << ", index_queue_elements: " << index_queue_elements
91
164
       << ", ttl_queue_size: " << ttl_queue_size << ", ttl_queue_elements: " << ttl_queue_elements
92
164
       << ", query_queue_size: " << query_queue_size
93
164
       << ", query_queue_elements: " << query_queue_elements << ", storage: " << storage;
94
164
    return ss.str();
95
164
}
96
97
FileCacheSettings get_file_cache_settings(size_t capacity, size_t max_query_cache_size,
98
                                          size_t normal_percent, size_t disposable_percent,
99
                                          size_t index_percent, size_t ttl_percent,
100
9
                                          const std::string& storage) {
101
9
    io::FileCacheSettings settings;
102
9
    if (capacity == 0) {
103
0
        return settings;
104
0
    }
105
9
    settings.capacity = capacity;
106
9
    settings.max_file_block_size = config::file_cache_each_block_size;
107
9
    settings.max_query_cache_size = max_query_cache_size;
108
9
    size_t per_size = settings.capacity / 100;
109
9
    settings.disposable_queue_size = per_size * disposable_percent;
110
9
    settings.disposable_queue_elements =
111
9
            std::max(settings.disposable_queue_size / settings.max_file_block_size,
112
9
                     REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
113
114
9
    settings.index_queue_size = per_size * index_percent;
115
9
    settings.index_queue_elements =
116
9
            std::max(settings.index_queue_size / settings.max_file_block_size,
117
9
                     REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
118
119
9
    settings.ttl_queue_size = per_size * ttl_percent;
120
9
    settings.ttl_queue_elements = std::max(settings.ttl_queue_size / settings.max_file_block_size,
121
9
                                           REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
122
123
9
    settings.query_queue_size = settings.capacity - settings.disposable_queue_size -
124
9
                                settings.index_queue_size - settings.ttl_queue_size;
125
9
    settings.query_queue_elements =
126
9
            std::max(settings.query_queue_size / settings.max_file_block_size,
127
9
                     REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
128
9
    settings.storage = storage;
129
9
    return settings;
130
9
}
131
132
591k
std::string UInt128Wrapper::to_string() const {
133
591k
    return get_hex_uint_lowercase(value_);
134
591k
}
135
136
FileBlocksHolderPtr FileCacheAllocatorBuilder::allocate_cache_holder(size_t offset, size_t size,
137
61.2k
                                                                     int64_t tablet_id) const {
138
61.2k
    CacheContext ctx;
139
18.4E
    ctx.cache_type = _expiration_time == 0 ? FileCacheType::NORMAL : FileCacheType::TTL;
140
61.2k
    ctx.expiration_time = _expiration_time;
141
61.2k
    ctx.is_cold_data = _is_cold_data;
142
61.2k
    ctx.tablet_id = tablet_id;
143
61.2k
    ReadStatistics stats;
144
61.2k
    ctx.stats = &stats;
145
61.2k
    auto holder = _cache->get_or_set(_cache_hash, offset, size, ctx);
146
61.2k
    return std::make_unique<FileBlocksHolder>(std::move(holder));
147
61.2k
}
148
149
template size_t LRUQueue::get_capacity(std::lock_guard<std::mutex>& cache_lock) const;
150
template void LRUQueue::remove(Iterator queue_it, std::lock_guard<std::mutex>& cache_lock);
151
152
0
std::string FileCacheInfo::to_string() const {
153
0
    std::stringstream ss;
154
0
    ss << "Hash: " << hash.to_string() << "\n"
155
0
       << "Expiration Time: " << expiration_time << "\n"
156
0
       << "Offset: " << offset << "\n"
157
0
       << "Cache Type: " << cache_type_to_string(cache_type) << "\n";
158
0
    return ss.str();
159
0
}
160
161
0
std::string InconsistencyType::to_string() const {
162
0
    std::string result = "Inconsistency Reason: ";
163
0
    if (type == NONE) {
164
0
        result += "NONE";
165
0
    } else {
166
0
        if (type & NOT_LOADED) {
167
0
            result += "NOT_LOADED ";
168
0
        }
169
0
        if (type & MISSING_IN_STORAGE) {
170
0
            result += "MISSING_IN_STORAGE ";
171
0
        }
172
0
        if (type & SIZE_INCONSISTENT) {
173
0
            result += "SIZE_INCONSISTENT ";
174
0
        }
175
0
        if (type & CACHE_TYPE_INCONSISTENT) {
176
0
            result += "CACHE_TYPE_INCONSISTENT ";
177
0
        }
178
0
        if (type & EXPIRATION_TIME_INCONSISTENT) {
179
0
            result += "EXPIRATION_TIME_INCONSISTENT ";
180
0
        }
181
0
        if (type & TMP_FILE_EXPECT_DOWNLOADING_STATE) {
182
0
            result += "TMP_FILE_EXPECT_DOWNLOADING_STATE";
183
0
        }
184
0
    }
185
0
    result += "\n";
186
0
    return result;
187
0
}
188
189
6.42M
std::optional<int64_t> get_tablet_id(std::string file_path) {
190
    // Expected path formats:
191
    // support both .dat and .idx file extensions
192
    // support formate see ut. storage_resource_test:StorageResourceTest.ParseTabletIdFromPath
193
194
6.42M
    if (file_path.empty()) {
195
0
        return std::nullopt;
196
0
    }
197
198
    // Find the position of "data/" in the path
199
6.42M
    std::string_view path_view = file_path;
200
6.42M
    std::string_view data_prefix = DATA_PREFIX;
201
6.42M
    size_t data_pos = path_view.find(data_prefix);
202
6.42M
    if (data_pos == std::string_view::npos) {
203
557
        return std::nullopt;
204
557
    }
205
206
6.42M
    if (data_prefix.length() + data_pos >= path_view.length()) {
207
1
        return std::nullopt;
208
1
    }
209
210
    // Extract the part after "data/"
211
6.42M
    path_view = path_view.substr(data_pos + data_prefix.length() + 1);
212
213
    // Check if path ends with .dat or .idx
214
6.42M
    if (!path_view.ends_with(".dat") && !path_view.ends_with(".idx")) {
215
530
        return std::nullopt;
216
530
    }
217
218
    // Count slashes in the remaining path
219
6.42M
    size_t slash_count = 0;
220
436M
    for (char c : path_view) {
221
436M
        if (c == '/') {
222
6.43M
            slash_count++;
223
6.43M
        }
224
436M
    }
225
226
    // Split path by '/'
227
6.42M
    std::vector<std::string_view> parts;
228
6.42M
    size_t start = 0;
229
6.42M
    size_t pos = 0;
230
12.8M
    while ((pos = path_view.find('/', start)) != std::string_view::npos) {
231
6.43M
        if (pos > start) {
232
6.42M
            parts.push_back(path_view.substr(start, pos - start));
233
6.42M
        }
234
6.43M
        start = pos + 1;
235
6.43M
    }
236
6.43M
    if (start < path_view.length()) {
237
6.43M
        parts.push_back(path_view.substr(start));
238
6.43M
    }
239
240
6.42M
    if (parts.empty()) {
241
0
        return std::nullopt;
242
0
    }
243
244
    // Determine path version based on slash count and extract tablet_id
245
    // Version 0: {tablet_id}/{rowset_id}_{seg_id}.dat (1 slash)
246
    // Version 1: {shard}/{tablet_id}/{rowset_id}/{seg_id}.dat (3 slashes)
247
248
6.42M
    if (slash_count == 1) {
249
        // Version 0 format: parts[0] should be tablet_id
250
6.43M
        if (parts.size() >= 1) {
251
6.43M
            try {
252
6.43M
                int64_t tablet_id = std::stoll(std::string(parts[0]));
253
6.43M
                return tablet_id;
254
6.43M
            } catch (const std::exception&) {
255
                // Not a valid number, return nullopt at last
256
0
            }
257
6.43M
        }
258
18.4E
    } else if (slash_count == 3) {
259
        // Version 1 format: parts[1] should be tablet_id (parts[0] is shard)
260
0
        if (parts.size() >= 2) {
261
0
            try {
262
0
                int64_t tablet_id = std::stoll(std::string(parts[1]));
263
0
                return tablet_id;
264
0
            } catch (const std::exception&) {
265
                // Not a valid number, return nullopt at last
266
0
            }
267
0
        }
268
0
    }
269
270
18.4E
    return std::nullopt;
271
6.42M
}
272
273
} // namespace doris::io