Coverage Report

Created: 2025-04-16 14:10

/root/doris/be/src/olap/tablet_meta.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/AgentService_types.h>
21
#include <gen_cpp/olap_file.pb.h>
22
#include <stdint.h>
23
24
#include <atomic>
25
#include <cstddef>
26
#include <limits>
27
#include <map>
28
#include <memory>
29
#include <mutex>
30
#include <optional>
31
#include <ostream>
32
#include <roaring/roaring.hh>
33
#include <shared_mutex>
34
#include <string>
35
#include <tuple>
36
#include <unordered_map>
37
#include <utility>
38
#include <vector>
39
40
#include "common/logging.h"
41
#include "common/status.h"
42
#include "gutil/stringprintf.h"
43
#include "io/fs/file_system.h"
44
#include "olap/binlog_config.h"
45
#include "olap/lru_cache.h"
46
#include "olap/metadata_adder.h"
47
#include "olap/olap_common.h"
48
#include "olap/rowset/rowset_meta.h"
49
#include "olap/tablet_schema.h"
50
#include "runtime/memory/lru_cache_policy.h"
51
#include "util/uid_util.h"
52
53
namespace json2pb {
54
#include "common/compile_check_begin.h"
55
struct Pb2JsonOptions;
56
} // namespace json2pb
57
58
namespace doris {
59
class TColumn;
60
61
// Lifecycle states that a Tablet can be in. Legal state transitions for a
62
// Tablet object:
63
//
64
//   NOTREADY -> RUNNING -> TOMBSTONED -> STOPPED -> SHUTDOWN
65
//      |           |            |          ^^^
66
//      |           |            +----------++|
67
//      |           +------------------------+|
68
//      +-------------------------------------+
69
70
enum TabletState {
71
    // Tablet is under alter table, rollup, clone
72
    TABLET_NOTREADY,
73
74
    TABLET_RUNNING,
75
76
    // Tablet integrity has been violated, such as missing versions.
77
    // In this state, tablet will not accept any incoming request.
78
    // Report this state to FE, scheduling BE to drop tablet.
79
    TABLET_TOMBSTONED,
80
81
    // Tablet is shutting down, files in disk still remained.
82
    TABLET_STOPPED,
83
84
    // Files have been removed, tablet has been shutdown completely.
85
    TABLET_SHUTDOWN
86
};
87
88
class DataDir;
89
class TabletMeta;
90
class DeleteBitmap;
91
class TBinlogConfig;
92
93
// Class encapsulates meta of tablet.
94
// The concurrency control is handled in Tablet Class, not in this class.
95
class TabletMeta : public MetadataAdder<TabletMeta> {
96
public:
97
    static TabletMetaSharedPtr create(
98
            const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t shard_id,
99
            uint32_t next_unique_id,
100
            const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id);
101
102
    TabletMeta();
103
    ~TabletMeta() override;
104
    TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id,
105
               int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema,
106
               uint32_t next_unique_id,
107
               const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
108
               TabletUid tablet_uid, TTabletType::type tabletType,
109
               TCompressionType::type compression_type, int64_t storage_policy_id = 0,
110
               bool enable_unique_key_merge_on_write = false,
111
               std::optional<TBinlogConfig> binlog_config = {},
112
               std::string compaction_policy = "size_based",
113
               int64_t time_series_compaction_goal_size_mbytes = 1024,
114
               int64_t time_series_compaction_file_count_threshold = 2000,
115
               int64_t time_series_compaction_time_threshold_seconds = 3600,
116
               int64_t time_series_compaction_empty_rowsets_threshold = 5,
117
               int64_t time_series_compaction_level_threshold = 1,
118
               TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
119
                       TInvertedIndexFileStorageFormat::V2);
120
    // If need add a filed in TableMeta, filed init copy in copy construct function
121
    TabletMeta(const TabletMeta& tablet_meta);
122
    TabletMeta(TabletMeta&& tablet_meta) = delete;
123
124
// UT
125
#ifdef BE_TEST
126
25
    TabletMeta(TabletSchemaSPtr tablet_schema) : _schema(tablet_schema) {}
127
#endif
128
129
    // Function create_from_file is used to be compatible with previous tablet_meta.
130
    // Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb.
131
    Status create_from_file(const std::string& file_path);
132
    static Status load_from_file(const std::string& file_path, TabletMetaPB* tablet_meta_pb);
133
    Status save(const std::string& file_path);
134
    Status save_as_json(const string& file_path);
135
    static Status save(const std::string& file_path, const TabletMetaPB& tablet_meta_pb);
136
    static std::string construct_header_file_path(const std::string& schema_hash_path,
137
                                                  int64_t tablet_id);
138
    Status save_meta(DataDir* data_dir);
139
140
    void serialize(std::string* meta_binary);
141
    Status deserialize(std::string_view meta_binary);
142
    void init_from_pb(const TabletMetaPB& tablet_meta_pb);
143
144
    void to_meta_pb(TabletMetaPB* tablet_meta_pb);
145
    void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options);
146
559
    size_t tablet_columns_num() const { return _schema->num_columns(); }
147
148
0
    TabletTypePB tablet_type() const { return _tablet_type; }
149
    TabletUid tablet_uid() const;
150
31
    void set_tablet_uid(TabletUid uid) { _tablet_uid = uid; }
151
    int64_t table_id() const;
152
    int64_t index_id() const;
153
    int64_t partition_id() const;
154
    int64_t tablet_id() const;
155
    int64_t replica_id() const;
156
0
    void set_replica_id(int64_t replica_id) { _replica_id = replica_id; }
157
    int32_t schema_hash() const;
158
    int32_t shard_id() const;
159
    void set_shard_id(int32_t shard_id);
160
    int64_t creation_time() const;
161
    void set_creation_time(int64_t creation_time);
162
    int64_t cumulative_layer_point() const;
163
    void set_cumulative_layer_point(int64_t new_point);
164
165
    size_t num_rows() const;
166
    // Disk space occupied by tablet, contain local and remote.
167
    size_t tablet_footprint() const;
168
    // Local disk space occupied by tablet.
169
    size_t tablet_local_size() const;
170
    // Remote disk space occupied by tablet.
171
    size_t tablet_remote_size() const;
172
173
    size_t tablet_local_index_size() const;
174
    size_t tablet_local_segment_size() const;
175
    size_t tablet_remote_index_size() const;
176
    size_t tablet_remote_segment_size() const;
177
178
    size_t version_count() const;
179
    size_t stale_version_count() const;
180
    size_t version_count_cross_with_range(const Version& range) const;
181
    Version max_version() const;
182
183
    TabletState tablet_state() const;
184
    void set_tablet_state(TabletState state);
185
186
    bool in_restore_mode() const;
187
    void set_in_restore_mode(bool in_restore_mode);
188
189
    const TabletSchemaSPtr& tablet_schema() const;
190
191
    TabletSchema* mutable_tablet_schema();
192
193
    const std::vector<RowsetMetaSharedPtr>& all_rs_metas() const;
194
    std::vector<RowsetMetaSharedPtr>& all_mutable_rs_metas();
195
    Status add_rs_meta(const RowsetMetaSharedPtr& rs_meta);
196
    void delete_rs_meta_by_version(const Version& version,
197
                                   std::vector<RowsetMetaSharedPtr>* deleted_rs_metas);
198
    // If same_version is true, the rowset in "to_delete" will not be added
199
    // to _stale_rs_meta, but to be deleted from rs_meta directly.
200
    void modify_rs_metas(const std::vector<RowsetMetaSharedPtr>& to_add,
201
                         const std::vector<RowsetMetaSharedPtr>& to_delete,
202
                         bool same_version = false);
203
    void revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas);
204
    void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap);
205
206
    const std::vector<RowsetMetaSharedPtr>& all_stale_rs_metas() const;
207
    RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version) const;
208
    void delete_stale_rs_meta_by_version(const Version& version);
209
    RowsetMetaSharedPtr acquire_stale_rs_meta_by_version(const Version& version) const;
210
211
    Status set_partition_id(int64_t partition_id);
212
213
380
    RowsetTypePB preferred_rowset_type() const { return _preferred_rowset_type; }
214
215
52
    void set_preferred_rowset_type(RowsetTypePB preferred_rowset_type) {
216
52
        _preferred_rowset_type = preferred_rowset_type;
217
52
    }
218
219
    // used for after tablet cloned to clear stale rowset
220
    void clear_stale_rowset();
221
222
    void clear_rowsets();
223
224
    // MUST hold EXCLUSIVE `_meta_lock` in belonged Tablet
225
    // `to_add` MUST NOT have overlapped version with `_rs_metas` in tablet meta.
226
    void add_rowsets_unchecked(const std::vector<RowsetSharedPtr>& to_add);
227
228
    bool all_beta() const;
229
230
22
    int64_t storage_policy_id() const { return _storage_policy_id; }
231
232
4
    void set_storage_policy_id(int64_t id) {
233
4
        VLOG_NOTICE << "set tablet_id : " << _table_id << " storage policy from "
234
0
                    << _storage_policy_id << " to " << id;
235
4
        _storage_policy_id = id;
236
4
    }
237
238
18
    UniqueId cooldown_meta_id() const { return _cooldown_meta_id; }
239
5
    void set_cooldown_meta_id(UniqueId uid) { _cooldown_meta_id = uid; }
240
241
    static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
242
                                         ColumnPB* column);
243
244
20
    DeleteBitmap& delete_bitmap() { return *_delete_bitmap; }
245
246
1.17k
    bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; }
247
248
    // TODO(Drogon): thread safety
249
0
    const BinlogConfig& binlog_config() const { return _binlog_config; }
250
0
    void set_binlog_config(BinlogConfig binlog_config) {
251
0
        _binlog_config = std::move(binlog_config);
252
0
    }
253
254
12
    void set_compaction_policy(std::string compaction_policy) {
255
12
        _compaction_policy = compaction_policy;
256
12
    }
257
1.56k
    std::string compaction_policy() const { return _compaction_policy; }
258
12
    void set_time_series_compaction_goal_size_mbytes(int64_t goal_size_mbytes) {
259
12
        _time_series_compaction_goal_size_mbytes = goal_size_mbytes;
260
12
    }
261
817
    int64_t time_series_compaction_goal_size_mbytes() const {
262
817
        return _time_series_compaction_goal_size_mbytes;
263
817
    }
264
12
    void set_time_series_compaction_file_count_threshold(int64_t file_count_threshold) {
265
12
        _time_series_compaction_file_count_threshold = file_count_threshold;
266
12
    }
267
814
    int64_t time_series_compaction_file_count_threshold() const {
268
814
        return _time_series_compaction_file_count_threshold;
269
814
    }
270
12
    void set_time_series_compaction_time_threshold_seconds(int64_t time_threshold) {
271
12
        _time_series_compaction_time_threshold_seconds = time_threshold;
272
12
    }
273
811
    int64_t time_series_compaction_time_threshold_seconds() const {
274
811
        return _time_series_compaction_time_threshold_seconds;
275
811
    }
276
0
    void set_time_series_compaction_empty_rowsets_threshold(int64_t empty_rowsets_threshold) {
277
0
        _time_series_compaction_empty_rowsets_threshold = empty_rowsets_threshold;
278
0
    }
279
813
    int64_t time_series_compaction_empty_rowsets_threshold() const {
280
813
        return _time_series_compaction_empty_rowsets_threshold;
281
813
    }
282
0
    void set_time_series_compaction_level_threshold(int64_t level_threshold) {
283
0
        _time_series_compaction_level_threshold = level_threshold;
284
0
    }
285
835
    int64_t time_series_compaction_level_threshold() const {
286
835
        return _time_series_compaction_level_threshold;
287
835
    }
288
289
331
    int64_t ttl_seconds() const {
290
331
        std::shared_lock rlock(_meta_lock);
291
331
        return _ttl_seconds;
292
331
    }
293
294
0
    void set_ttl_seconds(int64_t ttl_seconds) {
295
0
        std::lock_guard wlock(_meta_lock);
296
0
        _ttl_seconds = ttl_seconds;
297
0
    }
298
299
28
    int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; }
300
301
private:
302
    Status _save_meta(DataDir* data_dir);
303
    void _check_mow_rowset_cache_version_size(size_t rowset_cache_version_size);
304
305
    // _del_predicates is ignored to compare.
306
    friend bool operator==(const TabletMeta& a, const TabletMeta& b);
307
    friend bool operator!=(const TabletMeta& a, const TabletMeta& b);
308
309
private:
310
    int64_t _table_id = 0;
311
    int64_t _index_id = 0;
312
    int64_t _partition_id = 0;
313
    int64_t _tablet_id = 0;
314
    int64_t _replica_id = 0;
315
    int32_t _schema_hash = 0;
316
    int32_t _shard_id = 0;
317
    int64_t _creation_time = 0;
318
    int64_t _cumulative_layer_point = 0;
319
    TabletUid _tablet_uid;
320
    TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK;
321
322
    TabletState _tablet_state = TABLET_NOTREADY;
323
    // the reference of _schema may use in tablet, so here need keep
324
    // the lifetime of tablemeta and _schema is same with tablet
325
    TabletSchemaSPtr _schema;
326
    Cache::Handle* _handle = nullptr;
327
328
    std::vector<RowsetMetaSharedPtr> _rs_metas;
329
    // This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted.
330
    // These stale rowsets meta are been removed when rowsets' pathVersion is expired,
331
    // this policy is judged and computed by TimestampedVersionTracker.
332
    std::vector<RowsetMetaSharedPtr> _stale_rs_metas;
333
    bool _in_restore_mode = false;
334
    RowsetTypePB _preferred_rowset_type = BETA_ROWSET;
335
336
    // meta for cooldown
337
    int64_t _storage_policy_id = 0; // <= 0 means no storage policy
338
    UniqueId _cooldown_meta_id;
339
340
    // For unique key data model, the feature Merge-on-Write will leverage a primary
341
    // key index and a delete-bitmap to mark duplicate keys as deleted in load stage,
342
    // which can avoid the merging cost in read stage, and accelerate the aggregation
343
    // query performance significantly.
344
    bool _enable_unique_key_merge_on_write = false;
345
    std::shared_ptr<DeleteBitmap> _delete_bitmap;
346
347
    // binlog config
348
    BinlogConfig _binlog_config {};
349
350
    // meta for compaction
351
    std::string _compaction_policy;
352
    int64_t _time_series_compaction_goal_size_mbytes = 0;
353
    int64_t _time_series_compaction_file_count_threshold = 0;
354
    int64_t _time_series_compaction_time_threshold_seconds = 0;
355
    int64_t _time_series_compaction_empty_rowsets_threshold = 0;
356
    int64_t _time_series_compaction_level_threshold = 0;
357
358
    int64_t _avg_rs_meta_serialize_size = 0;
359
360
    // cloud
361
    int64_t _ttl_seconds = 0;
362
363
    mutable std::shared_mutex _meta_lock;
364
};
365
366
/**
367
 * Wraps multiple bitmaps for recording rows (row id) that are deleted or
368
 * overwritten. For now, it's only used when unique key merge-on-write property
369
 * enabled.
370
 *
371
 * RowsetId and SegmentId are for locating segment, Version here is a single
372
 * uint32_t means that at which "version" of the load causes the delete or
373
 * overwrite.
374
 *
375
 * The start and end version of a load is the same, it's ok and straightforward
376
 * to use a single uint32_t.
377
 *
378
 * e.g.
379
 * There is a key "key1" in rowset id 1, version [1,1], segment id 1, row id 1.
380
 * A new load also contains "key1", the rowset id 2, version [2,2], segment id 1
381
 * the delete bitmap will be `{1,1,2} -> 1`, which means the "row id 1" in
382
 * "rowset id 1, segment id 1" is deleted/overitten by some loads at "version 2"
383
 */
384
class DeleteBitmap {
385
public:
386
    mutable std::shared_mutex lock;
387
    mutable std::shared_mutex stale_delete_bitmap_lock;
388
    using SegmentId = uint32_t;
389
    using Version = uint64_t;
390
    using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
391
    std::map<BitmapKey, roaring::Roaring> delete_bitmap; // Ordered map
392
    constexpr static inline uint32_t INVALID_SEGMENT_ID = std::numeric_limits<uint32_t>::max() - 1;
393
    constexpr static inline uint32_t ROWSET_SENTINEL_MARK =
394
            std::numeric_limits<uint32_t>::max() - 1;
395
396
    // When a delete bitmap is merged into tablet's delete bitmap, the version of entries in the delete bitmap
397
    // will be replaced to the correspoding correct version. So before we finally merge a delete bitmap into
398
    // tablet's delete bitmap we can use arbitary version number in BitmapKey. Here we define some version numbers
399
    // for specific usage during this periods to avoid conflicts
400
    constexpr static inline uint64_t TEMP_VERSION_COMMON = 0;
401
402
    /**
403
     * 
404
     * @param tablet_id the tablet which this delete bitmap associates with
405
     */
406
    DeleteBitmap(int64_t tablet_id);
407
408
    /**
409
     * Copy c-tor for making delete bitmap snapshot on read path
410
     */
411
    DeleteBitmap(const DeleteBitmap& r);
412
    DeleteBitmap& operator=(const DeleteBitmap& r);
413
    /**
414
     * Move c-tor for making delete bitmap snapshot on read path
415
     */
416
    DeleteBitmap(DeleteBitmap&& r);
417
    DeleteBitmap& operator=(DeleteBitmap&& r);
418
419
    /**
420
     * Makes a snapshot of delete bitmap, read lock will be acquired in this
421
     * process
422
     */
423
    DeleteBitmap snapshot() const;
424
425
    /**
426
     * Makes a snapshot of delete bitmap on given version, read lock will be
427
     * acquired temporary in this process
428
     */
429
    DeleteBitmap snapshot(Version version) const;
430
431
    /**
432
     * Marks the specific row deleted
433
     */
434
    void add(const BitmapKey& bmk, uint32_t row_id);
435
436
    /**
437
     * Clears the deletetion mark specific row
438
     *
439
     * @return non-zero if the associated delete bitmap does not exist
440
     */
441
    int remove(const BitmapKey& bmk, uint32_t row_id);
442
443
    /**
444
     * Clears bitmaps in range [lower_key, upper_key)
445
     */
446
    void remove(const BitmapKey& lower_key, const BitmapKey& upper_key);
447
448
    /**
449
     * Checks if the given row is marked deleted
450
     *
451
     * @return true if marked deleted
452
     */
453
    bool contains(const BitmapKey& bmk, uint32_t row_id) const;
454
455
    /**
456
     * Checks if this delete bitmap is empty
457
     *
458
     * @return true if empty
459
     */
460
    bool empty() const;
461
462
    /**
463
     * return the total cardinality of the Delete Bitmap
464
     */
465
    uint64_t cardinality() const;
466
467
    /**
468
     * return the total size of the Delete Bitmap(after serialized)
469
     */
470
471
    uint64_t get_size() const;
472
473
    /**
474
     * Sets the bitmap of specific segment, it's may be insertion or replacement
475
     *
476
     * @return 1 if the insertion took place, 0 if the assignment took place
477
     */
478
    int set(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap);
479
480
    /**
481
     * Gets a copy of specific delete bmk
482
     *
483
     * @param segment_delete_bitmap output param
484
     * @return non-zero if the associated delete bitmap does not exist
485
     */
486
    int get(const BitmapKey& bmk, roaring::Roaring* segment_delete_bitmap) const;
487
488
    /**
489
     * Gets reference to a specific delete map, DO NOT use this function on a
490
     * mutable DeleteBitmap object
491
     * @return nullptr if the given bitmap does not exist
492
     */
493
    const roaring::Roaring* get(const BitmapKey& bmk) const;
494
495
    /**
496
     * Gets subset of delete_bitmap with given range [start, end)
497
     *
498
     * @parma start start
499
     * @parma end end
500
     * @parma subset_delete_map output param
501
     */
502
    void subset(const BitmapKey& start, const BitmapKey& end,
503
                DeleteBitmap* subset_delete_map) const;
504
505
    /**
506
     * Gets count of delete_bitmap with given range [start, end)
507
     *
508
     * @parma start start
509
     * @parma end end
510
     */
511
    size_t get_count_with_range(const BitmapKey& start, const BitmapKey& end) const;
512
513
    /**
514
     * Merges the given segment delete bitmap into *this
515
     *
516
     * @param bmk
517
     * @param segment_delete_bitmap
518
     */
519
    void merge(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap);
520
521
    /**
522
     * Merges the given delete bitmap into *this
523
     *
524
     * @param other
525
     */
526
    void merge(const DeleteBitmap& other);
527
528
    /**
529
     * Checks if the given row is marked deleted in bitmap with the condition:
530
     * all the bitmaps that
531
     * RowsetId and SegmentId are the same as the given ones,
532
     * and Version <= the given Version
533
     *
534
     * Note: aggregation cache may be used.
535
     *
536
     * @return true if marked deleted
537
     */
538
    bool contains_agg(const BitmapKey& bitmap, uint32_t row_id) const;
539
540
    bool contains_agg_without_cache(const BitmapKey& bmk, uint32_t row_id) const;
541
    /**
542
     * Gets aggregated delete_bitmap on rowset_id and version, the same effect:
543
     * `select sum(roaring::Roaring) where RowsetId=rowset_id and SegmentId=seg_id and Version <= version`
544
     *
545
     * @return shared_ptr to a bitmap, which may be empty
546
     */
547
    std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const;
548
    std::shared_ptr<roaring::Roaring> get_agg_without_cache(const BitmapKey& bmk) const;
549
550
    void remove_sentinel_marks();
551
552
    void add_to_remove_queue(const std::string& version_str,
553
                             const std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
554
                                                          DeleteBitmap::BitmapKey>>& vector);
555
    void remove_stale_delete_bitmap_from_queue(const std::vector<std::string>& vector);
556
557
    uint64_t get_delete_bitmap_count();
558
559
    bool has_calculated_for_multi_segments(const RowsetId& rowset_id) const;
560
561
    // return the size of the map
562
    size_t remove_rowset_cache_version(const RowsetId& rowset_id);
563
564
    void clear_rowset_cache_version();
565
566
    std::set<RowsetId> get_rowset_cache_version();
567
568
    class AggCachePolicy : public LRUCachePolicy {
569
    public:
570
        AggCachePolicy(size_t capacity)
571
                : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity,
572
                                 LRUCacheType::SIZE,
573
1
                                 config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {}
574
    };
575
576
    class AggCache {
577
    public:
578
        class Value : public LRUCacheValueBase {
579
        public:
580
            roaring::Roaring bitmap;
581
        };
582
583
1.01k
        AggCache(size_t size_in_bytes) {
584
1.01k
            static std::once_flag once;
585
1.01k
            std::call_once(once, [size_in_bytes] {
586
1
                auto* tmp = new AggCachePolicy(size_in_bytes);
587
1
                AggCache::s_repr.store(tmp, std::memory_order_release);
588
1
            });
589
590
1.01k
            while (!s_repr.load(std::memory_order_acquire)) {
591
0
            }
592
1.01k
        }
593
594
135
        static LRUCachePolicy* repr() { return s_repr.load(std::memory_order_acquire); }
595
        static std::atomic<AggCachePolicy*> s_repr;
596
    };
597
598
private:
599
    DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk) const;
600
601
    mutable std::shared_ptr<AggCache> _agg_cache;
602
    int64_t _tablet_id;
603
    mutable std::shared_mutex _rowset_cache_version_lock;
604
    mutable std::map<RowsetId, std::map<SegmentId, Version>> _rowset_cache_version;
605
    // <version, <tablet_id, BitmapKeyStart, BitmapKeyEnd>>
606
    std::map<std::string,
607
             std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>>
608
            _stale_delete_bitmap;
609
};
610
611
static const std::string SEQUENCE_COL = "__DORIS_SEQUENCE_COL__";
612
613
13.4k
inline TabletUid TabletMeta::tablet_uid() const {
614
13.4k
    return _tablet_uid;
615
13.4k
}
616
617
812
inline int64_t TabletMeta::table_id() const {
618
812
    return _table_id;
619
812
}
620
621
809
inline int64_t TabletMeta::index_id() const {
622
809
    return _index_id;
623
809
}
624
625
2.28k
inline int64_t TabletMeta::partition_id() const {
626
2.28k
    return _partition_id;
627
2.28k
}
628
629
19.4k
inline int64_t TabletMeta::tablet_id() const {
630
19.4k
    return _tablet_id;
631
19.4k
}
632
633
1.08k
inline int64_t TabletMeta::replica_id() const {
634
1.08k
    return _replica_id;
635
1.08k
}
636
637
2.83k
inline int32_t TabletMeta::schema_hash() const {
638
2.83k
    return _schema_hash;
639
2.83k
}
640
641
1.50k
inline int32_t TabletMeta::shard_id() const {
642
1.50k
    return _shard_id;
643
1.50k
}
644
645
17
inline void TabletMeta::set_shard_id(int32_t shard_id) {
646
17
    _shard_id = shard_id;
647
17
}
648
649
809
inline int64_t TabletMeta::creation_time() const {
650
809
    return _creation_time;
651
809
}
652
653
0
inline void TabletMeta::set_creation_time(int64_t creation_time) {
654
0
    _creation_time = creation_time;
655
0
}
656
657
809
inline int64_t TabletMeta::cumulative_layer_point() const {
658
809
    return _cumulative_layer_point;
659
809
}
660
661
0
inline void TabletMeta::set_cumulative_layer_point(int64_t new_point) {
662
0
    _cumulative_layer_point = new_point;
663
0
}
664
665
19
inline size_t TabletMeta::num_rows() const {
666
19
    size_t num_rows = 0;
667
32
    for (auto& rs : _rs_metas) {
668
32
        num_rows += rs->num_rows();
669
32
    }
670
19
    return num_rows;
671
19
}
672
673
8
inline size_t TabletMeta::tablet_footprint() const {
674
8
    size_t total_size = 0;
675
26
    for (auto& rs : _rs_metas) {
676
26
        total_size += rs->total_disk_size();
677
26
    }
678
8
    return total_size;
679
8
}
680
681
0
inline size_t TabletMeta::tablet_local_size() const {
682
0
    size_t total_size = 0;
683
0
    for (auto& rs : _rs_metas) {
684
0
        if (rs->is_local()) {
685
0
            total_size += rs->total_disk_size();
686
0
        }
687
0
    }
688
0
    return total_size;
689
0
}
690
691
0
inline size_t TabletMeta::tablet_remote_size() const {
692
0
    size_t total_size = 0;
693
0
    for (auto& rs : _rs_metas) {
694
0
        if (!rs->is_local()) {
695
0
            total_size += rs->total_disk_size();
696
0
        }
697
0
    }
698
0
    return total_size;
699
0
}
700
701
0
inline size_t TabletMeta::tablet_local_index_size() const {
702
0
    size_t total_size = 0;
703
0
    for (auto& rs : _rs_metas) {
704
0
        if (rs->is_local()) {
705
0
            total_size += rs->index_disk_size();
706
0
        }
707
0
    }
708
0
    return total_size;
709
0
}
710
711
0
inline size_t TabletMeta::tablet_local_segment_size() const {
712
0
    size_t total_size = 0;
713
0
    for (auto& rs : _rs_metas) {
714
0
        if (rs->is_local()) {
715
0
            total_size += rs->data_disk_size();
716
0
        }
717
0
    }
718
0
    return total_size;
719
0
}
720
721
0
inline size_t TabletMeta::tablet_remote_index_size() const {
722
0
    size_t total_size = 0;
723
0
    for (auto& rs : _rs_metas) {
724
0
        if (!rs->is_local()) {
725
0
            total_size += rs->index_disk_size();
726
0
        }
727
0
    }
728
0
    return total_size;
729
0
}
730
731
0
inline size_t TabletMeta::tablet_remote_segment_size() const {
732
0
    size_t total_size = 0;
733
0
    for (auto& rs : _rs_metas) {
734
0
        if (!rs->is_local()) {
735
0
            total_size += rs->data_disk_size();
736
0
        }
737
0
    }
738
0
    return total_size;
739
0
}
740
741
56
inline size_t TabletMeta::version_count() const {
742
56
    return _rs_metas.size();
743
56
}
744
745
28
inline size_t TabletMeta::stale_version_count() const {
746
28
    return _rs_metas.size();
747
28
}
748
749
10.2k
inline TabletState TabletMeta::tablet_state() const {
750
10.2k
    return _tablet_state;
751
10.2k
}
752
753
255
inline void TabletMeta::set_tablet_state(TabletState state) {
754
255
    _tablet_state = state;
755
255
}
756
757
809
inline bool TabletMeta::in_restore_mode() const {
758
809
    return _in_restore_mode;
759
809
}
760
761
0
inline void TabletMeta::set_in_restore_mode(bool in_restore_mode) {
762
0
    _in_restore_mode = in_restore_mode;
763
0
}
764
765
3.05k
inline const TabletSchemaSPtr& TabletMeta::tablet_schema() const {
766
3.05k
    return _schema;
767
3.05k
}
768
769
0
inline TabletSchema* TabletMeta::mutable_tablet_schema() {
770
0
    return _schema.get();
771
0
}
772
773
1.96k
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas() const {
774
1.96k
    return _rs_metas;
775
1.96k
}
776
777
0
inline std::vector<RowsetMetaSharedPtr>& TabletMeta::all_mutable_rs_metas() {
778
0
    return _rs_metas;
779
0
}
780
781
1.02k
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_stale_rs_metas() const {
782
1.02k
    return _stale_rs_metas;
783
1.02k
}
784
785
0
inline bool TabletMeta::all_beta() const {
786
0
    for (auto& rs : _rs_metas) {
787
0
        if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
788
0
            return false;
789
0
        }
790
0
    }
791
0
    for (auto& rs : _stale_rs_metas) {
792
0
        if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
793
0
            return false;
794
0
        }
795
0
    }
796
0
    return true;
797
0
}
798
799
std::string tablet_state_name(TabletState state);
800
801
// Only for unit test now.
802
bool operator==(const TabletMeta& a, const TabletMeta& b);
803
bool operator!=(const TabletMeta& a, const TabletMeta& b);
804
805
#include "common/compile_check_end.h"
806
} // namespace doris