Coverage Report

Created: 2026-07-13 13:44

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/segment/condition_cache.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <butil/macros.h>
21
#include <glog/logging.h>
22
#include <stddef.h>
23
#include <stdint.h>
24
25
#include <atomic>
26
#include <memory>
27
#include <roaring/roaring.hh>
28
#include <string>
29
#include <vector>
30
31
#include "common/config.h"
32
#include "common/status.h"
33
#include "io/fs/file_system.h"
34
#include "io/fs/path.h"
35
#include "runtime/exec_env.h"
36
#include "runtime/memory/lru_cache_policy.h"
37
#include "runtime/memory/mem_tracker.h"
38
#include "util/lru_cache.h"
39
#include "util/slice.h"
40
#include "util/time.h"
41
42
namespace doris {
43
44
// Context passed from scan/table-reader layers to physical readers for condition cache
45
// integration. On MISS, readers set filter_result[granule] to true when row-level predicates keep
46
// at least one row in that granule. On HIT, readers skip granules whose cached bit is false.
47
struct ConditionCacheContext {
48
    bool is_hit = false;
49
    std::shared_ptr<std::vector<bool>> filter_result; // per-granule: true = has surviving rows
50
    int64_t base_granule = 0;                         // global granule index of filter_result[0]
51
    size_t num_granules = 0; // authoritative bitmap length; excludes allocation-only guard bits
52
    static constexpr int GRANULE_SIZE = 2048;
53
};
54
55
namespace segment_v2 {
56
57
class ConditionCacheHandle;
58
59
class ConditionCache : public LRUCachePolicy {
60
public:
61
    using LRUCachePolicy::insert;
62
63
    // The cache key or segment lru cache
64
    struct CacheKey {
65
        CacheKey(RowsetId rowset_id_, int64_t segment_id_, uint64_t digest_)
66
2.99M
                : rowset_id(rowset_id_), segment_id(segment_id_), digest(digest_) {}
67
        RowsetId rowset_id;
68
        int64_t segment_id;
69
        uint64_t digest;
70
71
        // Encode to a flat binary which can be used as LRUCache's key
72
2.99M
        [[nodiscard]] std::string encode() const {
73
2.99M
            char buf[16];
74
2.99M
            memcpy(buf, &segment_id, 8);
75
2.99M
            memcpy(buf + 8, &digest, 8);
76
77
2.99M
            return rowset_id.to_string() + std::string(buf, 16);
78
2.99M
        }
79
    };
80
81
    class CacheValue : public LRUCacheValueBase {
82
    public:
83
        std::shared_ptr<std::vector<bool>> filter_result;
84
        // The bitmap coordinate system is part of the cached result. A later scan may prune a
85
        // different first row group, so it must not derive this origin from its current plan.
86
        int64_t base_granule = 0;
87
    };
88
89
    // Cache key for external tables (Hive ORC/Parquet)
90
    struct ExternalCacheKey {
91
        static constexpr uint8_t BASE_GRANULE_AWARE_VERSION = 1;
92
93
60.8k
        ExternalCacheKey() = default;
94
        ExternalCacheKey(const std::string& path_, int64_t modification_time_, int64_t file_size_,
95
                         uint64_t digest_, int64_t start_offset_, int64_t size_,
96
                         uint8_t format_version_ = 0)
97
24.7k
                : path(path_),
98
24.7k
                  modification_time(modification_time_),
99
24.7k
                  file_size(file_size_),
100
24.7k
                  digest(digest_),
101
24.7k
                  start_offset(start_offset_),
102
24.7k
                  size(size_),
103
24.7k
                  format_version(format_version_) {}
104
        std::string path;
105
        int64_t modification_time = 0;
106
        int64_t file_size = 0;
107
        uint64_t digest = 0;
108
        int64_t start_offset = 0;
109
        int64_t size = 0;
110
        uint8_t format_version = 0;
111
112
29.0k
        [[nodiscard]] std::string encode() const {
113
29.0k
            std::string key = path;
114
29.0k
            char buf[41];
115
29.0k
            memcpy(buf, &modification_time, 8);
116
29.0k
            memcpy(buf + 8, &file_size, 8);
117
29.0k
            memcpy(buf + 16, &digest, 8);
118
29.0k
            memcpy(buf + 24, &start_offset, 8);
119
29.0k
            memcpy(buf + 32, &size, 8);
120
29.0k
            buf[40] = static_cast<char>(format_version);
121
29.0k
            key.append(buf, 41);
122
29.0k
            return key;
123
29.0k
        }
124
    };
125
126
    // Create global instance of this class
127
18
    static ConditionCache* create_global_cache(size_t capacity, uint32_t num_shards = 16) {
128
18
        auto* res = new ConditionCache(capacity, num_shards);
129
18
        return res;
130
18
    }
131
132
    // Return global instance.
133
    // Client should call create_global_cache before.
134
3.02M
    static ConditionCache* instance() { return ExecEnv::GetInstance()->get_condition_cache(); }
135
136
    ConditionCache() = delete;
137
138
    ConditionCache(size_t capacity, uint32_t num_shards)
139
18
            : LRUCachePolicy(CachePolicy::CacheType::CONDITION_CACHE, capacity, LRUCacheType::SIZE,
140
18
                             config::inverted_index_cache_stale_sweep_time_sec, num_shards,
141
18
                             /*element_count_capacity*/ 0, /*enable_prune*/ true,
142
18
                             /*is_lru_k*/ true) {}
143
144
    template <typename KeyType>
145
    bool lookup(const KeyType& key, ConditionCacheHandle* handle);
146
147
    template <typename KeyType>
148
    void insert(const KeyType& key, std::shared_ptr<std::vector<bool>> filter_result,
149
                int64_t base_granule = 0);
150
};
151
152
class ConditionCacheHandle {
153
public:
154
1.61M
    ConditionCacheHandle() = default;
155
156
    ConditionCacheHandle(LRUCachePolicy* cache, Cache::Handle* handle)
157
1.58M
            : _cache(cache), _handle(handle) {}
158
159
3.19M
    ~ConditionCacheHandle() {
160
3.19M
        if (_handle != nullptr) {
161
1.58M
            _cache->release(_handle);
162
1.58M
        }
163
3.19M
    }
164
165
0
    ConditionCacheHandle(ConditionCacheHandle&& other) noexcept {
166
0
        // we can use std::exchange if we switch c++14 on
167
0
        std::swap(_cache, other._cache);
168
0
        std::swap(_handle, other._handle);
169
0
    }
170
171
159k
    ConditionCacheHandle& operator=(ConditionCacheHandle&& other) noexcept {
172
159k
        std::swap(_cache, other._cache);
173
159k
        std::swap(_handle, other._handle);
174
159k
        return *this;
175
159k
    }
176
177
0
    LRUCachePolicy* cache() const { return _cache; }
178
179
159k
    std::shared_ptr<std::vector<bool>> get_filter_result() const {
180
159k
        if (!_cache) {
181
0
            return nullptr;
182
0
        }
183
159k
        return ((ConditionCache::CacheValue*)_cache->value(_handle))->filter_result;
184
159k
    }
185
186
3.14k
    int64_t get_base_granule() const {
187
3.14k
        DORIS_CHECK(_cache != nullptr);
188
3.14k
        return ((ConditionCache::CacheValue*)_cache->value(_handle))->base_granule;
189
3.14k
    }
190
191
private:
192
    LRUCachePolicy* _cache = nullptr;
193
    Cache::Handle* _handle = nullptr;
194
195
    // Don't allow copy and assign
196
    DISALLOW_COPY_AND_ASSIGN(ConditionCacheHandle);
197
};
198
199
} // namespace segment_v2
200
} // namespace doris