Coverage Report

Created: 2026-07-08 18:37

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/fs/file_meta_cache.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "io/fs/file_meta_cache.h"
19
20
#include <utility>
21
22
#include "common/config.h"
23
#include "common/logging.h"
24
#include "util/stopwatch.hpp"
25
26
namespace doris {
27
namespace {
28
29
2
void update_profile_counter(int64_t* counter, int64_t value = 1) {
30
2
    if (counter != nullptr) {
31
2
        *counter += value;
32
2
    }
33
2
}
34
35
} // namespace
36
37
FileMetaCache::FileMetaCache(int64_t capacity,
38
                             std::unique_ptr<FileMetaPersistentCache> persistent_cache)
39
43
        : _cache(capacity), _persistent_cache(std::move(persistent_cache)) {}
40
41
std::string FileMetaCache::get_key(const std::string file_name, int64_t modification_time,
42
228
                                   int64_t file_size) {
43
228
    std::string meta_cache_key;
44
228
    meta_cache_key.resize(file_name.size() + sizeof(int64_t));
45
46
228
    memcpy(meta_cache_key.data(), file_name.data(), file_name.size());
47
228
    if (modification_time != 0) {
48
8
        memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t));
49
220
    } else {
50
220
        memcpy(meta_cache_key.data() + file_name.size(), &file_size, sizeof(int64_t));
51
220
    }
52
228
    return meta_cache_key;
53
228
}
54
55
std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader,
56
216
                                   const io::FileDescription& _file_description) {
57
216
    return FileMetaCache::get_key(
58
216
            file_reader->path().native(), _file_description.mtime,
59
216
            _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size);
60
216
}
61
62
0
bool FileMetaCache::is_persistent_cache_enabled() {
63
0
    return config::enable_external_file_meta_disk_cache &&
64
0
           config::external_file_meta_disk_cache_max_entry_bytes > 0;
65
0
}
66
67
1
bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) {
68
1
    const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes;
69
1
    return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 &&
70
1
           payload_size <= static_cast<uint64_t>(max_entry_bytes);
71
1
}
72
73
FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context,
74
                                                ObjLRUCache::CacheHandle* handle,
75
                                                std::string* serialized_meta,
76
1
                                                FileMetaCacheProfile* profile) {
77
1
    DCHECK(handle != nullptr);
78
1
    DCHECK(serialized_meta != nullptr);
79
1
    if (context.enable_memory_cache && lookup(context.key, handle)) {
80
1
        serialized_meta->clear();
81
1
        if (profile != nullptr) {
82
1
            update_profile_counter(profile->hit_cache);
83
1
            update_profile_counter(profile->hit_memory_cache);
84
1
        }
85
1
        return {.state = FileMetaCacheLookupState::MEMORY_HIT};
86
1
    }
87
88
0
    FileMetaCacheLookupResult result;
89
0
    int64_t persisted_read_time = 0;
90
0
    if (lookup_persistent_cache(context, serialized_meta, &persisted_read_time)) {
91
0
        result.state = FileMetaCacheLookupState::PERSISTED_HIT;
92
0
        if (profile != nullptr) {
93
0
            update_profile_counter(profile->hit_cache);
94
0
            update_profile_counter(profile->hit_disk_cache);
95
0
            update_profile_counter(profile->read_disk_cache_time, persisted_read_time);
96
0
        }
97
0
    } else if (is_persistent_cache_enabled() && profile != nullptr) {
98
0
        update_profile_counter(profile->miss_disk_cache);
99
0
    }
100
0
    return result;
101
1
}
102
103
0
void FileMetaCache::invalidate_persistent_cache(const FileMetaCacheContext& context) {
104
0
    if (_persistent_cache != nullptr) {
105
0
        _persistent_cache->remove(context.format, context.key);
106
0
    }
107
0
}
108
109
bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context,
110
0
                                            std::string* payload, int64_t* read_time) {
111
0
    DCHECK(payload != nullptr);
112
0
    DCHECK(read_time != nullptr);
113
0
    payload->clear();
114
0
    *read_time = 0;
115
0
    if (!is_persistent_cache_enabled() || _persistent_cache == nullptr) {
116
0
        return false;
117
0
    }
118
119
0
    MonotonicStopWatch watch;
120
0
    watch.start();
121
0
    Status status = _persistent_cache->read(context.format, context.key, context.modification_time,
122
0
                                            context.file_size, payload);
123
0
    *read_time = watch.elapsed_time();
124
0
    if (!status.ok()) {
125
0
        payload->clear();
126
0
        VLOG_DEBUG << "lookup file meta persistent cache failed: " << status;
127
0
        return false;
128
0
    }
129
0
    return true;
130
0
}
131
132
bool FileMetaCache::insert_persistent_cache(const FileMetaCacheContext& context,
133
1
                                            std::string_view payload, int64_t* write_time) {
134
1
    DCHECK(write_time != nullptr);
135
1
    *write_time = 0;
136
1
    if (!is_persistent_cache_payload_size_allowed(static_cast<uint64_t>(payload.size())) ||
137
1
        _persistent_cache == nullptr) {
138
1
        return false;
139
1
    }
140
141
0
    MonotonicStopWatch watch;
142
0
    watch.start();
143
0
    Status status = _persistent_cache->write(context.format, context.key, context.modification_time,
144
0
                                             context.file_size, payload);
145
0
    *write_time = watch.elapsed_time();
146
0
    if (!status.ok()) {
147
0
        VLOG_DEBUG << "insert file meta persistent cache failed: " << status;
148
0
        return false;
149
0
    }
150
0
    return true;
151
0
}
152
153
} // namespace doris