Coverage Report

Created: 2026-03-13 09:37

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/tablet/tablet_meta.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/AgentService_types.h>
21
#include <gen_cpp/FrontendService_types.h>
22
#include <gen_cpp/olap_file.pb.h>
23
#include <stdint.h>
24
25
#include <atomic>
26
#include <cstddef>
27
#include <limits>
28
#include <map>
29
#include <memory>
30
#include <mutex>
31
#include <optional>
32
#include <ostream>
33
#include <roaring/roaring.hh>
34
#include <shared_mutex>
35
#include <string>
36
#include <tuple>
37
#include <unordered_map>
38
#include <utility>
39
#include <vector>
40
41
#include "common/logging.h"
42
#include "common/status.h"
43
#include "io/fs/file_system.h"
44
#include "runtime/memory/lru_cache_policy.h"
45
#include "storage/binlog_config.h"
46
#include "storage/metadata_adder.h"
47
#include "storage/olap_common.h"
48
#include "storage/rowset/rowset_meta.h"
49
#include "storage/tablet/tablet_schema.h"
50
#include "util/lru_cache.h"
51
#include "util/uid_util.h"
52
53
namespace json2pb {
54
#include "common/compile_check_begin.h"
55
struct Pb2JsonOptions;
56
} // namespace json2pb
57
58
namespace doris {
59
class TColumn;
60
61
// Lifecycle states that a Tablet can be in. Legal state transitions for a
62
// Tablet object:
63
//
64
//   NOTREADY -> RUNNING -> TOMBSTONED -> STOPPED -> SHUTDOWN
65
//      |           |            |          ^^^
66
//      |           |            +----------++|
67
//      |           +------------------------+|
68
//      +-------------------------------------+
69
70
enum TabletState {
71
    // Tablet is under alter table, rollup, clone
72
    TABLET_NOTREADY,
73
74
    TABLET_RUNNING,
75
76
    // Tablet integrity has been violated, such as missing versions.
77
    // In this state, tablet will not accept any incoming request.
78
    // Report this state to FE, scheduling BE to drop tablet.
79
    TABLET_TOMBSTONED,
80
81
    // Tablet is shutting down, files in disk still remained.
82
    TABLET_STOPPED,
83
84
    // Files have been removed, tablet has been shutdown completely.
85
    TABLET_SHUTDOWN
86
};
87
88
class DataDir;
89
class TabletMeta;
90
class DeleteBitmap;
91
class TBinlogConfig;
92
93
// Class encapsulates meta of tablet.
94
// The concurrency control is handled in Tablet Class, not in this class.
95
class TabletMeta : public MetadataAdder<TabletMeta> {
96
public:
97
    static TabletMetaSharedPtr create(
98
            const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t shard_id,
99
            uint32_t next_unique_id,
100
            const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id);
101
102
    TabletMeta();
103
    ~TabletMeta() override;
104
    TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id,
105
               int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema,
106
               uint32_t next_unique_id,
107
               const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
108
               TabletUid tablet_uid, TTabletType::type tabletType,
109
               TCompressionType::type compression_type, int64_t storage_policy_id = 0,
110
               bool enable_unique_key_merge_on_write = false,
111
               std::optional<TBinlogConfig> binlog_config = {},
112
               std::string compaction_policy = "size_based",
113
               int64_t time_series_compaction_goal_size_mbytes = 1024,
114
               int64_t time_series_compaction_file_count_threshold = 2000,
115
               int64_t time_series_compaction_time_threshold_seconds = 3600,
116
               int64_t time_series_compaction_empty_rowsets_threshold = 5,
117
               int64_t time_series_compaction_level_threshold = 1,
118
               TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
119
                       TInvertedIndexFileStorageFormat::V2,
120
               TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT,
121
               TStorageFormat::type storage_format = TStorageFormat::V2,
122
               int32_t vertical_compaction_num_columns_per_group = 5);
123
    // If need add a filed in TableMeta, filed init copy in copy construct function
124
    TabletMeta(const TabletMeta& tablet_meta);
125
    TabletMeta(TabletMeta&& tablet_meta) = delete;
126
127
// UT
128
#ifdef BE_TEST
129
    TabletMeta(TabletSchemaSPtr tablet_schema) : _schema(tablet_schema) {}
130
#endif
131
132
    // Function create_from_file is used to be compatible with previous tablet_meta.
133
    // Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb.
134
    Status create_from_file(const std::string& file_path);
135
    // Used to create tablet meta from memory buffer.
136
    Status create_from_buffer(const uint8_t* buffer, size_t buffer_size);
137
    static Status load_from_file(const std::string& file_path, TabletMetaPB* tablet_meta_pb);
138
    Status save(const std::string& file_path);
139
    Status save_as_json(const std::string& file_path);
140
    static Status save(const std::string& file_path, const TabletMetaPB& tablet_meta_pb);
141
    static std::string construct_header_file_path(const std::string& schema_hash_path,
142
                                                  int64_t tablet_id);
143
    Status save_meta(DataDir* data_dir);
144
145
    void serialize(std::string* meta_binary);
146
    Status deserialize(std::string_view meta_binary);
147
    void init_from_pb(const TabletMetaPB& tablet_meta_pb);
148
149
    void to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_meta);
150
    void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options);
151
366k
    size_t tablet_columns_num() const { return _schema->num_columns(); }
152
153
0
    TabletTypePB tablet_type() const { return _tablet_type; }
154
    TabletUid tablet_uid() const;
155
206
    void set_tablet_uid(TabletUid uid) { _tablet_uid = uid; }
156
    int64_t table_id() const;
157
    int64_t index_id() const;
158
    int64_t partition_id() const;
159
    int64_t tablet_id() const;
160
    int64_t replica_id() const;
161
0
    void set_replica_id(int64_t replica_id) { _replica_id = replica_id; }
162
    int32_t schema_hash() const;
163
    int32_t shard_id() const;
164
    void set_shard_id(int32_t shard_id);
165
    int64_t creation_time() const;
166
    void set_creation_time(int64_t creation_time);
167
    int64_t cumulative_layer_point() const;
168
    void set_cumulative_layer_point(int64_t new_point);
169
170
    size_t num_rows() const;
171
    // Disk space occupied by tablet, contain local and remote.
172
    size_t tablet_footprint() const;
173
    // Local disk space occupied by tablet.
174
    size_t tablet_local_size() const;
175
    // Remote disk space occupied by tablet.
176
    size_t tablet_remote_size() const;
177
178
    size_t tablet_local_index_size() const;
179
    size_t tablet_local_segment_size() const;
180
    size_t tablet_remote_index_size() const;
181
    size_t tablet_remote_segment_size() const;
182
183
    size_t version_count() const;
184
    size_t stale_version_count() const;
185
    size_t version_count_cross_with_range(const Version& range) const;
186
    Version max_version() const;
187
188
    TabletState tablet_state() const;
189
    void set_tablet_state(TabletState state);
190
191
    bool in_restore_mode() const;
192
    void set_in_restore_mode(bool in_restore_mode);
193
194
    const TabletSchemaSPtr& tablet_schema() const;
195
196
    TabletSchema* mutable_tablet_schema();
197
198
    const RowsetMetaMapContainer& all_rs_metas() const;
199
    RowsetMetaMapContainer& all_mutable_rs_metas();
200
    Status add_rs_meta(const RowsetMetaSharedPtr& rs_meta);
201
    void delete_rs_meta_by_version(const Version& version,
202
                                   std::vector<RowsetMetaSharedPtr>* deleted_rs_metas);
203
    // If same_version is true, the rowset in "to_delete" will not be added
204
    // to _stale_rs_meta, but to be deleted from rs_meta directly.
205
    void modify_rs_metas(const std::vector<RowsetMetaSharedPtr>& to_add,
206
                         const std::vector<RowsetMetaSharedPtr>& to_delete,
207
                         bool same_version = false);
208
    void revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas);
209
    void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap);
210
211
    const RowsetMetaMapContainer& all_stale_rs_metas() const;
212
    RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version) const;
213
    void delete_stale_rs_meta_by_version(const Version& version);
214
    RowsetMetaSharedPtr acquire_stale_rs_meta_by_version(const Version& version) const;
215
216
    Status set_partition_id(int64_t partition_id);
217
218
22.8k
    RowsetTypePB preferred_rowset_type() const { return _preferred_rowset_type; }
219
220
6.08k
    void set_preferred_rowset_type(RowsetTypePB preferred_rowset_type) {
221
6.08k
        _preferred_rowset_type = preferred_rowset_type;
222
6.08k
    }
223
224
    // used for after tablet cloned to clear stale rowset
225
    void clear_stale_rowset();
226
227
    void clear_rowsets();
228
229
    // MUST hold EXCLUSIVE `_meta_lock` in belonged Tablet
230
    // `to_add` MUST NOT have overlapped version with `_rs_metas` in tablet meta.
231
    void add_rowsets_unchecked(const std::vector<RowsetSharedPtr>& to_add);
232
233
    bool all_beta() const;
234
235
2.01M
    int64_t storage_policy_id() const { return _storage_policy_id; }
236
237
4
    void set_storage_policy_id(int64_t id) {
238
4
        VLOG_NOTICE << "set tablet_id : " << _table_id << " storage policy from "
239
4
                    << _storage_policy_id << " to " << id;
240
4
        _storage_policy_id = id;
241
4
    }
242
243
3.23M
    UniqueId cooldown_meta_id() const { return _cooldown_meta_id; }
244
5
    void set_cooldown_meta_id(UniqueId uid) { _cooldown_meta_id = uid; }
245
246
    static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
247
                                         ColumnPB* column);
248
249
3.01M
    DeleteBitmapPtr delete_bitmap_ptr() { return _delete_bitmap; }
250
1.00M
    DeleteBitmap& delete_bitmap() { return *_delete_bitmap; }
251
    void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version);
252
253
6.67M
    bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; }
254
#ifdef BE_TEST
255
    void set_enable_unique_key_merge_on_write(bool value) {
256
        _enable_unique_key_merge_on_write = value;
257
    }
258
#endif
259
    // TODO(Drogon): thread safety
260
14.4k
    const BinlogConfig& binlog_config() const { return _binlog_config; }
261
0
    void set_binlog_config(BinlogConfig binlog_config) {
262
0
        _binlog_config = std::move(binlog_config);
263
0
    }
264
265
13
    void set_compaction_policy(std::string compaction_policy) {
266
13
        _compaction_policy = compaction_policy;
267
13
    }
268
518M
    std::string compaction_policy() const { return _compaction_policy; }
269
12
    void set_time_series_compaction_goal_size_mbytes(int64_t goal_size_mbytes) {
270
12
        _time_series_compaction_goal_size_mbytes = goal_size_mbytes;
271
12
    }
272
174k
    int64_t time_series_compaction_goal_size_mbytes() const {
273
174k
        return _time_series_compaction_goal_size_mbytes;
274
174k
    }
275
12
    void set_time_series_compaction_file_count_threshold(int64_t file_count_threshold) {
276
12
        _time_series_compaction_file_count_threshold = file_count_threshold;
277
12
    }
278
174k
    int64_t time_series_compaction_file_count_threshold() const {
279
174k
        return _time_series_compaction_file_count_threshold;
280
174k
    }
281
13
    void set_time_series_compaction_time_threshold_seconds(int64_t time_threshold) {
282
13
        _time_series_compaction_time_threshold_seconds = time_threshold;
283
13
    }
284
174k
    int64_t time_series_compaction_time_threshold_seconds() const {
285
174k
        return _time_series_compaction_time_threshold_seconds;
286
174k
    }
287
0
    void set_time_series_compaction_empty_rowsets_threshold(int64_t empty_rowsets_threshold) {
288
0
        _time_series_compaction_empty_rowsets_threshold = empty_rowsets_threshold;
289
0
    }
290
174k
    int64_t time_series_compaction_empty_rowsets_threshold() const {
291
174k
        return _time_series_compaction_empty_rowsets_threshold;
292
174k
    }
293
0
    void set_time_series_compaction_level_threshold(int64_t level_threshold) {
294
0
        _time_series_compaction_level_threshold = level_threshold;
295
0
    }
296
174k
    int64_t time_series_compaction_level_threshold() const {
297
174k
        return _time_series_compaction_level_threshold;
298
174k
    }
299
300
0
    void set_vertical_compaction_num_columns_per_group(int32_t num) {
301
0
        _vertical_compaction_num_columns_per_group = num;
302
0
    }
303
185k
    int32_t vertical_compaction_num_columns_per_group() const {
304
185k
        return _vertical_compaction_num_columns_per_group;
305
185k
    }
306
307
1.71M
    int64_t ttl_seconds() const {
308
1.71M
        std::shared_lock rlock(_meta_lock);
309
1.71M
        return _ttl_seconds;
310
1.71M
    }
311
312
    void set_ttl_seconds(int64_t ttl_seconds) {
313
        std::lock_guard wlock(_meta_lock);
314
        _ttl_seconds = ttl_seconds;
315
    }
316
317
14.9k
    int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; }
318
319
347k
    EncryptionAlgorithmPB encryption_algorithm() const { return _encryption_algorithm; }
320
321
private:
322
    Status _save_meta(DataDir* data_dir);
323
    void _check_mow_rowset_cache_version_size(size_t rowset_cache_version_size);
324
325
    // _del_predicates is ignored to compare.
326
    friend bool operator==(const TabletMeta& a, const TabletMeta& b);
327
    friend bool operator!=(const TabletMeta& a, const TabletMeta& b);
328
329
private:
330
    int64_t _table_id = 0;
331
    int64_t _index_id = 0;
332
    int64_t _partition_id = 0;
333
    int64_t _tablet_id = 0;
334
    int64_t _replica_id = 0;
335
    int32_t _schema_hash = 0;
336
    int32_t _shard_id = 0;
337
    int64_t _creation_time = 0;
338
    int64_t _cumulative_layer_point = 0;
339
    TabletUid _tablet_uid;
340
    TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK;
341
342
    TabletState _tablet_state = TABLET_NOTREADY;
343
    // the reference of _schema may use in tablet, so here need keep
344
    // the lifetime of tablemeta and _schema is same with tablet
345
    TabletSchemaSPtr _schema;
346
    Cache::Handle* _handle = nullptr;
347
348
    RowsetMetaMapContainer _rs_metas;
349
    // This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted.
350
    // These stale rowsets meta are been removed when rowsets' pathVersion is expired,
351
    // this policy is judged and computed by TimestampedVersionTracker.
352
    RowsetMetaMapContainer _stale_rs_metas;
353
    bool _in_restore_mode = false;
354
    RowsetTypePB _preferred_rowset_type = BETA_ROWSET;
355
356
    // meta for cooldown
357
    int64_t _storage_policy_id = 0; // <= 0 means no storage policy
358
    UniqueId _cooldown_meta_id;
359
360
    // For unique key data model, the feature Merge-on-Write will leverage a primary
361
    // key index and a delete-bitmap to mark duplicate keys as deleted in load stage,
362
    // which can avoid the merging cost in read stage, and accelerate the aggregation
363
    // query performance significantly.
364
    bool _enable_unique_key_merge_on_write = false;
365
    std::shared_ptr<DeleteBitmap> _delete_bitmap;
366
367
    // binlog config
368
    BinlogConfig _binlog_config {};
369
370
    // meta for compaction
371
    std::string _compaction_policy;
372
    int64_t _time_series_compaction_goal_size_mbytes = 0;
373
    int64_t _time_series_compaction_file_count_threshold = 0;
374
    int64_t _time_series_compaction_time_threshold_seconds = 0;
375
    int64_t _time_series_compaction_empty_rowsets_threshold = 0;
376
    int64_t _time_series_compaction_level_threshold = 0;
377
    int32_t _vertical_compaction_num_columns_per_group = 5;
378
379
    int64_t _avg_rs_meta_serialize_size = 0;
380
381
    // cloud
382
    int64_t _ttl_seconds = 0;
383
384
    EncryptionAlgorithmPB _encryption_algorithm = PLAINTEXT;
385
386
    // Persisted storage format for this tablet (e.g. V2, V3). Used to derive
387
    // schema-level defaults such as external ColumnMeta usage.
388
    TStorageFormat::type _storage_format = TStorageFormat::V2;
389
390
    mutable std::shared_mutex _meta_lock;
391
};
392
393
class DeleteBitmapAggCache : public LRUCachePolicy {
394
public:
395
    DeleteBitmapAggCache(size_t capacity);
396
397
    static DeleteBitmapAggCache* instance();
398
399
    static DeleteBitmapAggCache* create_instance(size_t capacity);
400
401
    DeleteBitmap snapshot(int64_t tablet_id);
402
403
    class Value : public LRUCacheValueBase {
404
    public:
405
        roaring::Roaring bitmap;
406
    };
407
};
408
409
/**
410
 * Wraps multiple bitmaps for recording rows (row id) that are deleted or
411
 * overwritten. For now, it's only used when unique key merge-on-write property
412
 * enabled.
413
 *
414
 * RowsetId and SegmentId are for locating segment, Version here is a single
415
 * uint32_t means that at which "version" of the load causes the delete or
416
 * overwrite.
417
 *
418
 * The start and end version of a load is the same, it's ok and straightforward
419
 * to use a single uint32_t.
420
 *
421
 * e.g.
422
 * There is a key "key1" in rowset id 1, version [1,1], segment id 1, row id 1.
423
 * A new load also contains "key1", the rowset id 2, version [2,2], segment id 1
424
 * the delete bitmap will be `{1,1,2} -> 1`, which means the "row id 1" in
425
 * "rowset id 1, segment id 1" is deleted/overitten by some loads at "version 2"
426
 */
427
class DeleteBitmap {
428
public:
429
    mutable std::shared_mutex lock;
430
    using SegmentId = uint32_t;
431
    using Version = uint64_t;
432
    using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
433
    std::map<BitmapKey, roaring::Roaring> delete_bitmap; // Ordered map
434
    constexpr static inline uint32_t INVALID_SEGMENT_ID = std::numeric_limits<uint32_t>::max() - 1;
435
    constexpr static inline uint32_t ROWSET_SENTINEL_MARK =
436
            std::numeric_limits<uint32_t>::max() - 1;
437
438
    // When a delete bitmap is merged into tablet's delete bitmap, the version of entries in the delete bitmap
439
    // will be replaced to the correspoding correct version. So before we finally merge a delete bitmap into
440
    // tablet's delete bitmap we can use arbitary version number in BitmapKey. Here we define some version numbers
441
    // for specific usage during this periods to avoid conflicts
442
    constexpr static inline uint64_t TEMP_VERSION_COMMON = 0;
443
444
    /**
445
     * 
446
     * @param tablet_id the tablet which this delete bitmap associates with
447
     */
448
    DeleteBitmap(int64_t tablet_id);
449
450
    /**
451
     * Copy c-tor for making delete bitmap snapshot on read path
452
     */
453
    DeleteBitmap(const DeleteBitmap& r);
454
    DeleteBitmap& operator=(const DeleteBitmap& r);
455
    /**
456
     * Move c-tor for making delete bitmap snapshot on read path
457
     */
458
    DeleteBitmap(DeleteBitmap&& r) noexcept;
459
    DeleteBitmap& operator=(DeleteBitmap&& r) noexcept;
460
461
    static DeleteBitmap from_pb(const DeleteBitmapPB& pb, int64_t tablet_id);
462
463
    DeleteBitmapPB to_pb();
464
465
    /**
466
     * Makes a snapshot of delete bitmap, read lock will be acquired in this
467
     * process
468
     */
469
    DeleteBitmap snapshot() const;
470
471
    /**
472
     * Makes a snapshot of delete bitmap on given version, read lock will be
473
     * acquired temporary in this process
474
     */
475
    DeleteBitmap snapshot(Version version) const;
476
477
    /**
478
     * Marks the specific row deleted
479
     */
480
    void add(const BitmapKey& bmk, uint32_t row_id);
481
482
    /**
483
     * Clears the deletetion mark specific row
484
     *
485
     * @return non-zero if the associated delete bitmap does not exist
486
     */
487
    int remove(const BitmapKey& bmk, uint32_t row_id);
488
489
    /**
490
     * Clears bitmaps in range [lower_key, upper_key)
491
     */
492
    void remove(const BitmapKey& lower_key, const BitmapKey& upper_key);
493
    void remove(const std::vector<std::tuple<BitmapKey, BitmapKey>>& key_ranges);
494
495
    /**
496
     * Checks if the given row is marked deleted
497
     *
498
     * @return true if marked deleted
499
     */
500
    bool contains(const BitmapKey& bmk, uint32_t row_id) const;
501
502
    /**
503
     * Checks if this delete bitmap is empty
504
     *
505
     * @return true if empty
506
     */
507
    bool empty() const;
508
509
    /**
510
     * return the total cardinality of the Delete Bitmap
511
     */
512
    uint64_t cardinality() const;
513
514
    /**
515
     * return the total size of the Delete Bitmap(after serialized)
516
     */
517
518
    uint64_t get_size() const;
519
520
    /**
521
     * Sets the bitmap of specific segment, it's may be insertion or replacement
522
     *
523
     * @return 1 if the insertion took place, 0 if the assignment took place
524
     */
525
    int set(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap);
526
527
    /**
528
     * Gets a copy of specific delete bmk
529
     *
530
     * @param segment_delete_bitmap output param
531
     * @return non-zero if the associated delete bitmap does not exist
532
     */
533
    int get(const BitmapKey& bmk, roaring::Roaring* segment_delete_bitmap) const;
534
535
    /**
536
     * Gets reference to a specific delete map, DO NOT use this function on a
537
     * mutable DeleteBitmap object
538
     * @return nullptr if the given bitmap does not exist
539
     */
540
    const roaring::Roaring* get(const BitmapKey& bmk) const;
541
542
    /**
543
     * Gets subset of delete_bitmap with given range [start, end)
544
     *
545
     * @parma start start
546
     * @parma end end
547
     * @parma subset_delete_map output param
548
     */
549
    void subset(const BitmapKey& start, const BitmapKey& end,
550
                DeleteBitmap* subset_delete_map) const;
551
    void subset(std::vector<std::pair<RowsetId, int64_t>>& rowset_ids, int64_t start_version,
552
                int64_t end_version, DeleteBitmap* subset_delete_map) const;
553
554
    /**
555
     * Gets subset of delete_bitmap of the input rowsets
556
     * with given version range [start_version, end_version] and agg to end_version,
557
     * then merge to subset_delete_map
558
     */
559
    void subset_and_agg(std::vector<std::pair<RowsetId, int64_t>>& rowset_ids,
560
                        int64_t start_version, int64_t end_version,
561
                        DeleteBitmap* subset_delete_map) const;
562
563
    /**
564
     * Gets count of delete_bitmap with given range [start, end)
565
     *
566
     * @parma start start
567
     * @parma end end
568
     */
569
    size_t get_count_with_range(const BitmapKey& start, const BitmapKey& end) const;
570
571
    /**
572
     * Merges the given segment delete bitmap into *this
573
     *
574
     * @param bmk
575
     * @param segment_delete_bitmap
576
     */
577
    void merge(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap);
578
579
    /**
580
     * Merges the given delete bitmap into *this
581
     *
582
     * @param other
583
     */
584
    void merge(const DeleteBitmap& other);
585
586
    /**
587
     * Checks if the given row is marked deleted in bitmap with the condition:
588
     * all the bitmaps that
589
     * RowsetId and SegmentId are the same as the given ones,
590
     * and Version <= the given Version
591
     *
592
     * Note: aggregation cache may be used.
593
     *
594
     * @return true if marked deleted
595
     */
596
    bool contains_agg(const BitmapKey& bitmap, uint32_t row_id) const;
597
598
    bool contains_agg_with_cache_if_eligible(const BitmapKey& bmk, uint32_t row_id) const;
599
    /**
600
     * Gets aggregated delete_bitmap on rowset_id and version, the same effect:
601
     * `select sum(roaring::Roaring) where RowsetId=rowset_id and SegmentId=seg_id and Version <= version`
602
     *
603
     * @return shared_ptr to a bitmap, which may be empty
604
     */
605
    std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const;
606
    std::shared_ptr<roaring::Roaring> get_agg_without_cache(const BitmapKey& bmk,
607
                                                            const int64_t start_version = 0) const;
608
609
    void remove_sentinel_marks();
610
611
    uint64_t get_delete_bitmap_count();
612
613
    void traverse_rowset_and_version(
614
            const std::function<int(const RowsetId& rowsetId, int64_t version)>& func) const;
615
616
    bool has_calculated_for_multi_segments(const RowsetId& rowset_id) const;
617
618
    // return the size of the map
619
    size_t remove_rowset_cache_version(const RowsetId& rowset_id);
620
621
    void clear_rowset_cache_version();
622
623
    std::set<std::string> get_rowset_cache_version();
624
625
    DeleteBitmap agg_cache_snapshot();
626
627
    void set_tablet_id(int64_t tablet_id);
628
629
    /**
630
     * Calculate diffset with given `key_set`. All entries with keys contained in this delete bitmap but not
631
     * in given key_set will be added to the output delete bitmap.
632
     *
633
     * @return Deletebitmap containning all entries in diffset
634
    */
635
    DeleteBitmap diffset(const std::set<BitmapKey>& key_set) const;
636
637
private:
638
    DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk) const;
639
640
    int64_t _tablet_id;
641
    mutable std::shared_mutex _rowset_cache_version_lock;
642
    mutable std::map<RowsetId, std::map<SegmentId, Version>> _rowset_cache_version;
643
};
644
645
2.85M
inline TabletUid TabletMeta::tablet_uid() const {
646
2.85M
    return _tablet_uid;
647
2.85M
}
648
649
1.06M
inline int64_t TabletMeta::table_id() const {
650
1.06M
    return _table_id;
651
1.06M
}
652
653
1.30M
inline int64_t TabletMeta::index_id() const {
654
1.30M
    return _index_id;
655
1.30M
}
656
657
4.75M
inline int64_t TabletMeta::partition_id() const {
658
4.75M
    return _partition_id;
659
4.75M
}
660
661
1.82G
inline int64_t TabletMeta::tablet_id() const {
662
1.82G
    return _tablet_id;
663
1.82G
}
664
665
1.47M
inline int64_t TabletMeta::replica_id() const {
666
1.47M
    return _replica_id;
667
1.47M
}
668
669
2.16M
inline int32_t TabletMeta::schema_hash() const {
670
2.16M
    return _schema_hash;
671
2.16M
}
672
673
430k
inline int32_t TabletMeta::shard_id() const {
674
430k
    return _shard_id;
675
430k
}
676
677
189
inline void TabletMeta::set_shard_id(int32_t shard_id) {
678
189
    _shard_id = shard_id;
679
189
}
680
681
234k
inline int64_t TabletMeta::creation_time() const {
682
234k
    return _creation_time;
683
234k
}
684
685
inline void TabletMeta::set_creation_time(int64_t creation_time) {
686
    _creation_time = creation_time;
687
}
688
689
67.9k
inline int64_t TabletMeta::cumulative_layer_point() const {
690
67.9k
    return _cumulative_layer_point;
691
67.9k
}
692
693
0
inline void TabletMeta::set_cumulative_layer_point(int64_t new_point) {
694
0
    _cumulative_layer_point = new_point;
695
0
}
696
697
1.25M
inline size_t TabletMeta::num_rows() const {
698
1.25M
    size_t num_rows = 0;
699
2.28M
    for (const auto& [_, rs] : _rs_metas) {
700
2.28M
        num_rows += rs->num_rows();
701
2.28M
    }
702
1.25M
    return num_rows;
703
1.25M
}
704
705
22.0k
inline size_t TabletMeta::tablet_footprint() const {
706
22.0k
    size_t total_size = 0;
707
61.6k
    for (const auto& [_, rs] : _rs_metas) {
708
61.6k
        total_size += rs->total_disk_size();
709
61.6k
    }
710
22.0k
    return total_size;
711
22.0k
}
712
713
4.15M
inline size_t TabletMeta::tablet_local_size() const {
714
4.15M
    size_t total_size = 0;
715
7.57M
    for (const auto& [_, rs] : _rs_metas) {
716
7.57M
        if (rs->is_local()) {
717
7.54M
            total_size += rs->total_disk_size();
718
7.54M
        }
719
7.57M
    }
720
4.15M
    return total_size;
721
4.15M
}
722
723
4.15M
inline size_t TabletMeta::tablet_remote_size() const {
724
4.15M
    size_t total_size = 0;
725
7.57M
    for (const auto& [_, rs] : _rs_metas) {
726
7.57M
        if (!rs->is_local()) {
727
29.5k
            total_size += rs->total_disk_size();
728
29.5k
        }
729
7.57M
    }
730
4.15M
    return total_size;
731
4.15M
}
732
733
1.25M
inline size_t TabletMeta::tablet_local_index_size() const {
734
1.25M
    size_t total_size = 0;
735
2.28M
    for (const auto& [_, rs] : _rs_metas) {
736
2.28M
        if (rs->is_local()) {
737
2.28M
            total_size += rs->index_disk_size();
738
2.28M
        }
739
2.28M
    }
740
1.25M
    return total_size;
741
1.25M
}
742
743
1.25M
inline size_t TabletMeta::tablet_local_segment_size() const {
744
1.25M
    size_t total_size = 0;
745
2.28M
    for (const auto& [_, rs] : _rs_metas) {
746
2.28M
        if (rs->is_local()) {
747
2.28M
            total_size += rs->data_disk_size();
748
2.28M
        }
749
2.28M
    }
750
1.25M
    return total_size;
751
1.25M
}
752
753
1.25M
inline size_t TabletMeta::tablet_remote_index_size() const {
754
1.25M
    size_t total_size = 0;
755
2.28M
    for (const auto& [_, rs] : _rs_metas) {
756
2.28M
        if (!rs->is_local()) {
757
14
            total_size += rs->index_disk_size();
758
14
        }
759
2.28M
    }
760
1.25M
    return total_size;
761
1.25M
}
762
763
1.25M
inline size_t TabletMeta::tablet_remote_segment_size() const {
764
1.25M
    size_t total_size = 0;
765
2.28M
    for (const auto& [_, rs] : _rs_metas) {
766
2.28M
        if (!rs->is_local()) {
767
14
            total_size += rs->data_disk_size();
768
14
        }
769
2.28M
    }
770
1.25M
    return total_size;
771
1.25M
}
772
773
9.67M
inline size_t TabletMeta::version_count() const {
774
9.67M
    return _rs_metas.size();
775
9.67M
}
776
777
14.9k
inline size_t TabletMeta::stale_version_count() const {
778
14.9k
    return _rs_metas.size();
779
14.9k
}
780
781
489M
inline TabletState TabletMeta::tablet_state() const {
782
489M
    return _tablet_state;
783
489M
}
784
785
22.8k
inline void TabletMeta::set_tablet_state(TabletState state) {
786
22.8k
    _tablet_state = state;
787
22.8k
}
788
789
68.0k
inline bool TabletMeta::in_restore_mode() const {
790
68.0k
    return _in_restore_mode;
791
68.0k
}
792
793
0
inline void TabletMeta::set_in_restore_mode(bool in_restore_mode) {
794
0
    _in_restore_mode = in_restore_mode;
795
0
}
796
797
784M
inline const TabletSchemaSPtr& TabletMeta::tablet_schema() const {
798
784M
    return _schema;
799
784M
}
800
801
5
inline TabletSchema* TabletMeta::mutable_tablet_schema() {
802
5
    return _schema.get();
803
5
}
804
805
11.5M
inline const RowsetMetaMapContainer& TabletMeta::all_rs_metas() const {
806
11.5M
    return _rs_metas;
807
11.5M
}
808
809
336k
inline RowsetMetaMapContainer& TabletMeta::all_mutable_rs_metas() {
810
336k
    return _rs_metas;
811
336k
}
812
813
854k
inline const RowsetMetaMapContainer& TabletMeta::all_stale_rs_metas() const {
814
854k
    return _stale_rs_metas;
815
854k
}
816
817
0
inline bool TabletMeta::all_beta() const {
818
0
    for (const auto& [_, rs] : _rs_metas) {
819
0
        if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
820
0
            return false;
821
0
        }
822
0
    }
823
0
    for (const auto& [_, rs] : _stale_rs_metas) {
824
0
        if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
825
0
            return false;
826
0
        }
827
0
    }
828
0
    return true;
829
0
}
830
831
std::string tablet_state_name(TabletState state);
832
833
// Only for unit test now.
834
bool operator==(const TabletMeta& a, const TabletMeta& b);
835
bool operator!=(const TabletMeta& a, const TabletMeta& b);
836
837
#include "common/compile_check_end.h"
838
} // namespace doris