Coverage Report

Created: 2026-05-09 15:32

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/tablet/tablet.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <butil/macros.h>
21
#include <glog/logging.h>
22
23
#include <atomic>
24
#include <cstddef>
25
#include <cstdint>
26
#include <functional>
27
#include <memory>
28
#include <mutex>
29
#include <ostream>
30
#include <set>
31
#include <shared_mutex>
32
#include <string>
33
#include <string_view>
34
#include <utility>
35
#include <vector>
36
37
#include "common/atomic_shared_ptr.h"
38
#include "common/config.h"
39
#include "common/metrics/metrics.h"
40
#include "common/status.h"
41
#include "storage/binlog.h"
42
#include "storage/binlog_config.h"
43
#include "storage/data_dir.h"
44
#include "storage/olap_common.h"
45
#include "storage/partial_update_info.h"
46
#include "storage/rowset/rowset.h"
47
#include "storage/rowset/rowset_meta.h"
48
#include "storage/rowset/rowset_reader.h"
49
#include "storage/segment/segment.h"
50
#include "storage/segment/segment_loader.h"
51
#include "storage/tablet/base_tablet.h"
52
#include "storage/version_graph.h"
53
#include "util/once.h"
54
#include "util/slice.h"
55
56
namespace bvar {
57
template <typename T>
58
class Adder;
59
}
60
61
namespace doris {
62
63
class Tablet;
64
class CumulativeCompactionPolicy;
65
class CompactionMixin;
66
class SingleReplicaCompaction;
67
class RowsetWriter;
68
struct RowsetWriterContext;
69
class TTabletInfo;
70
class TabletMetaPB;
71
class TupleDescriptor;
72
class CalcDeleteBitmapToken;
73
enum CompressKind : int;
74
class RowsetBinlogMetasPB;
75
76
namespace io {
77
class RemoteFileSystem;
78
} // namespace io
79
class Block;
80
struct RowLocation;
81
enum KeysType : int;
82
enum SortType : int;
83
84
enum TabletStorageType { STORAGE_TYPE_LOCAL, STORAGE_TYPE_REMOTE, STORAGE_TYPE_REMOTE_AND_LOCAL };
85
86
extern bvar::Adder<uint64_t> unused_remote_rowset_num;
87
88
static inline constexpr auto TRACE_TABLET_LOCK_THRESHOLD = std::chrono::seconds(1);
89
90
struct WriteCooldownMetaExecutors {
91
    WriteCooldownMetaExecutors(size_t executor_nums = 5);
92
93
    void stop();
94
95
    void submit(TabletSharedPtr tablet);
96
5
    size_t _get_executor_pos(int64_t tablet_id) const {
97
5
        return std::hash<int64_t>()(tablet_id) % _executor_nums;
98
5
    };
99
    // Each executor is a mpsc to ensure uploads of the same tablet meta are not concurrent
100
    // FIXME(AlexYue): Use mpsc instead of `ThreadPool` with 1 thread
101
    // We use PriorityThreadPool since it would call status inside it's `shutdown` function.
102
    // Consider one situation where the StackTraceCache's singleton is detructed before
103
    // this WriteCooldownMetaExecutors's singleton, then invoking the status would also call
104
    // StackTraceCache which would then result in heap use after free like #23834
105
    std::vector<std::unique_ptr<PriorityThreadPool>> _executors;
106
    std::unordered_set<int64_t> _pending_tablets;
107
    std::mutex _latch;
108
    size_t _executor_nums;
109
};
110
111
class Tablet final : public BaseTablet {
112
public:
113
    Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* data_dir,
114
           const std::string_view& cumulative_compaction_type = "");
115
116
3.82k
    DataDir* data_dir() const { return _data_dir; }
117
277
    int64_t replica_id() const { return _tablet_meta->replica_id(); }
118
119
15.2k
    std::string tablet_path() const override { return _tablet_path; }
120
3
    std::string row_binlog_path() const {
121
3
        return fmt::format("{}/{}", _tablet_path, FDRowBinlogSuffix);
122
3
    }
123
3.17k
    std::string get_rowset_path(const RowsetMetaSharedPtr& rowset_meta) const {
124
3.17k
        if (!rowset_meta->is_local()) {
125
10
            return "";
126
10
        }
127
3.16k
        return rowset_meta->is_row_binlog() ? row_binlog_path() : tablet_path();
128
3.17k
    }
129
130
    bool set_tablet_schema_into_rowset_meta();
131
    Status init();
132
    bool init_succeeded();
133
134
    bool is_used();
135
136
    void register_tablet_into_dir();
137
    void deregister_tablet_from_dir();
138
139
    void save_meta();
140
    // Used in clone task, to update local meta when finishing a clone job
141
    Status revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add,
142
                              const std::vector<RowsetSharedPtr>& to_delete,
143
                              bool is_incremental_clone);
144
145
    int64_t cumulative_layer_point() const;
146
    void set_cumulative_layer_point(int64_t new_point);
147
    inline int64_t cumulative_promotion_size() const;
148
    inline void set_cumulative_promotion_size(int64_t new_size);
149
150
    // Disk space occupied by tablet, contain local and remote.
151
    size_t tablet_footprint() override;
152
    // Local disk space occupied by tablet.
153
    size_t tablet_local_size();
154
    // Remote disk space occupied by tablet.
155
    size_t tablet_remote_size();
156
157
    size_t num_rows();
158
    size_t version_count() const;
159
    size_t stale_version_count() const;
160
    bool exceed_version_limit(int32_t limit) override;
161
    uint64_t segment_count() const;
162
    Version max_version() const;
163
    CumulativeCompactionPolicy* cumulative_compaction_policy();
164
165
    // properties encapsulated in TabletSchema
166
    SortType sort_type() const;
167
    size_t sort_col_num() const;
168
    size_t num_columns() const;
169
    size_t num_null_columns() const;
170
    size_t num_short_key_columns() const;
171
    size_t num_rows_per_row_block() const;
172
    double bloom_filter_fpp() const;
173
    size_t next_unique_id() const;
174
    int64_t avg_rs_meta_serialize_size() const;
175
176
    // operation in rowsets
177
    Status add_rowset(RowsetSharedPtr rowset, RowsetSharedPtr row_binlog_rowset = nullptr);
178
    Status create_initial_rowset(const int64_t version);
179
180
    // MUST hold EXCLUSIVE `_meta_lock`.
181
    Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
182
                          std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false);
183
    bool rowset_exists_unlocked(const RowsetSharedPtr& rowset);
184
185
    Status add_inc_rowset(const RowsetSharedPtr& rowset);
186
    /// Delete stale rowset by timing. This delete policy uses now() minutes
187
    /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset
188
    /// to delete.  When rowset is deleted, it will be added to StorageEngine unused map and record
189
    /// need to delete flag.
190
    void delete_expired_stale_rowset();
191
192
    // if quiet is true, no error log will be printed if there are missing versions
193
    Status check_version_integrity(const Version& version, bool quiet = false);
194
    bool check_version_exist(const Version& version) const;
195
    void acquire_version_and_rowsets(
196
            std::vector<std::pair<Version, RowsetSharedPtr>>* version_rowsets) const;
197
198
    // If skip_missing_version is true, skip versions if they are missing.
199
    Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits,
200
                              const CaptureRowsetOps& opts) override;
201
202
    // Find the missed versions until the spec_version.
203
    //
204
    // for example:
205
    //     [0-4][5-5][8-8][9-9][14-14]
206
    // if spec_version = 12, it will return [6, 6], [7, 7], [10, 10], [11, 11], [12, 12]
207
    Versions calc_missed_versions(int64_t spec_version, Versions existing_versions) const override;
208
209
    // meta lock
210
777
    std::shared_mutex& get_header_lock() { return _meta_lock; }
211
2
    std::mutex& get_rowset_update_lock() { return _rowset_update_lock; }
212
33
    std::mutex& get_push_lock() { return _ingest_lock; }
213
24
    std::mutex& get_base_compaction_lock() { return _base_compaction_lock; }
214
303
    std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; }
215
1
    std::shared_mutex& get_meta_store_lock() { return _meta_store_lock; }
216
217
56
    std::shared_timed_mutex& get_migration_lock() { return _migration_lock; }
218
219
22
    std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; }
220
221
    // operation for compaction
222
    bool can_do_compaction(size_t path_hash, CompactionType compaction_type);
223
    bool suitable_for_compaction(
224
            CompactionType compaction_type,
225
            std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy);
226
227
    uint32_t calc_compaction_score();
228
229
    // This function to find max continuous version from the beginning.
230
    // For example: If there are 1, 2, 3, 5, 6, 7 versions belongs tablet, then 3 is target.
231
    // 3 will be saved in "version", and 7 will be saved in "max_version", if max_version != nullptr
232
    void max_continuous_version_from_beginning(Version* version, Version* max_version = nullptr);
233
234
0
    void set_bad(bool is_bad) { _is_bad = is_bad; }
235
236
257
    int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; }
237
1
    void set_last_cumu_compaction_failure_time(int64_t millis) {
238
1
        _last_cumu_compaction_failure_millis = millis;
239
1
    }
240
241
0
    int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; }
242
0
    void set_last_base_compaction_failure_time(int64_t millis) {
243
0
        _last_base_compaction_failure_millis = millis;
244
0
    }
245
246
0
    int64_t last_full_compaction_failure_time() { return _last_full_compaction_failure_millis; }
247
0
    void set_last_full_compaction_failure_time(int64_t millis) {
248
0
        _last_full_compaction_failure_millis = millis;
249
0
    }
250
251
6
    int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; }
252
2
    void set_last_cumu_compaction_success_time(int64_t millis) {
253
2
        _last_cumu_compaction_success_millis = millis;
254
2
    }
255
256
0
    int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; }
257
0
    void set_last_base_compaction_success_time(int64_t millis) {
258
0
        _last_base_compaction_success_millis = millis;
259
0
    }
260
261
0
    int64_t last_full_compaction_success_time() { return _last_full_compaction_success_millis; }
262
0
    void set_last_full_compaction_success_time(int64_t millis) {
263
0
        _last_full_compaction_success_millis = millis;
264
0
    }
265
266
0
    int64_t last_cumu_compaction_schedule_time() { return _last_cumu_compaction_schedule_millis; }
267
0
    void set_last_cumu_compaction_schedule_time(int64_t millis) {
268
0
        _last_cumu_compaction_schedule_millis = millis;
269
0
    }
270
271
0
    int64_t last_base_compaction_schedule_time() { return _last_base_compaction_schedule_millis; }
272
0
    void set_last_base_compaction_schedule_time(int64_t millis) {
273
0
        _last_base_compaction_schedule_millis = millis;
274
0
    }
275
276
0
    int64_t last_full_compaction_schedule_time() { return _last_full_compaction_schedule_millis; }
277
0
    void set_last_full_compaction_schedule_time(int64_t millis) {
278
0
        _last_full_compaction_schedule_millis = millis;
279
0
    }
280
281
0
    void set_last_single_compaction_failure_status(std::string status) {
282
0
        _last_single_compaction_failure_status = std::move(status);
283
0
    }
284
285
0
    void set_last_fetched_version(Version version) { _last_fetched_version = std::move(version); }
286
287
    void delete_all_files();
288
289
    void check_tablet_path_exists();
290
291
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_cumulative_compaction();
292
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction();
293
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction();
294
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_build_inverted_index(
295
            const std::set<int64_t>& alter_index_uids, bool is_drop_op);
296
297
    // used for single compaction to get the local versions
298
    // Single compaction does not require remote rowsets and cannot violate the cooldown semantics
299
    std::vector<Version> get_all_local_versions();
300
301
    void calculate_cumulative_point();
302
    // TODO(ygl):
303
0
    bool is_primary_replica() { return false; }
304
305
    // return true if the checkpoint is actually done
306
    bool do_tablet_meta_checkpoint();
307
308
    // Check whether the rowset is useful or not, unuseful rowset can be swept up then.
309
    // Rowset which is under tablet's management is useful, i.e. rowset is in
310
    // _rs_version_map, or _stale_rs_version_map.
311
    // Rowset whose version range is not covered by this tablet is also useful.
312
    bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta);
313
314
    void build_tablet_report_info(TTabletInfo* tablet_info,
315
                                  bool enable_consecutive_missing_check = false,
316
                                  bool enable_path_check = false);
317
318
    // return a json string to show the compaction status of this tablet
319
    void get_compaction_status(std::string* json_result);
320
321
    static Status prepare_compaction_and_calculate_permits(
322
            CompactionType compaction_type, const TabletSharedPtr& tablet,
323
            std::shared_ptr<CompactionMixin>& compaction, int64_t& permits);
324
325
    void execute_compaction(CompactionMixin& compaction);
326
    void execute_single_replica_compaction(SingleReplicaCompaction& compaction);
327
328
    void set_cumulative_compaction_policy(
329
0
            std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) {
330
0
        _cumulative_compaction_policy = cumulative_compaction_policy;
331
0
    }
332
333
0
    std::shared_ptr<CumulativeCompactionPolicy> get_cumulative_compaction_policy() {
334
0
        return _cumulative_compaction_policy;
335
0
    }
336
337
1
    void set_last_cumu_compaction_status(std::string status) {
338
1
        _last_cumu_compaction_status = std::move(status);
339
1
    }
340
341
0
    std::string get_last_cumu_compaction_status() { return _last_cumu_compaction_status; }
342
343
0
    void set_last_base_compaction_status(std::string status) {
344
0
        _last_base_compaction_status = std::move(status);
345
0
    }
346
347
0
    std::string get_last_base_compaction_status() { return _last_base_compaction_status; }
348
349
0
    void set_last_full_compaction_status(std::string status) {
350
0
        _last_full_compaction_status = std::move(status);
351
0
    }
352
353
0
    std::string get_last_full_compaction_status() { return _last_full_compaction_status; }
354
355
    std::tuple<int64_t, int64_t> get_visible_version_and_time() const;
356
357
568
    void set_visible_version(const std::shared_ptr<const VersionWithTime>& visible_version) {
358
568
        _visible_version.store(visible_version);
359
568
    }
360
361
    bool should_fetch_from_peer();
362
363
0
    inline bool all_beta() const {
364
0
        std::shared_lock rdlock(_meta_lock);
365
0
        return _tablet_meta->all_beta();
366
0
    }
367
368
0
    const TabletSchemaSPtr& tablet_schema_unlocked() const { return _max_version_schema; }
369
370
    Result<std::unique_ptr<RowsetWriter>> create_rowset_writer(RowsetWriterContext& context,
371
                                                               bool vertical) override;
372
373
    Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer(
374
            const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info,
375
            int64_t txn_expiration = 0) override;
376
    Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer(
377
            RowsetWriterContext& context, const RowsetId& rowset_id);
378
379
    Status create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset);
380
381
    // MUST hold EXCLUSIVE `_meta_lock`
382
    void add_rowsets(const std::vector<RowsetSharedPtr>& to_add);
383
    // MUST hold EXCLUSIVE `_meta_lock`
384
    Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool move_to_stale);
385
386
    // MUST hold SHARED `_meta_lock`
387
489
    const auto& rowset_map() const { return _rs_version_map; }
388
    // MUST hold SHARED `_meta_lock`
389
484
    const auto& stale_rowset_map() const { return _stale_rs_version_map; }
390
    // MUST hold SHARED `_meta_lock`
391
0
    const auto& row_binlog_rowset_map() const { return _row_binlog_rs_version_map; }
392
393
    ////////////////////////////////////////////////////////////////////////////
394
    // begin cooldown functions
395
    ////////////////////////////////////////////////////////////////////////////
396
22
    int64_t storage_policy_id() const { return _tablet_meta->storage_policy_id(); }
397
4
    void set_storage_policy_id(int64_t id) { _tablet_meta->set_storage_policy_id(id); }
398
399
0
    int64_t last_failed_follow_cooldown_time() const { return _last_failed_follow_cooldown_time; }
400
401
    // Cooldown to remote fs.
402
    Status cooldown(RowsetSharedPtr rowset = nullptr);
403
404
    RowsetSharedPtr pick_cooldown_rowset();
405
406
    RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size);
407
408
    struct CooldownConf {
409
        int64_t term = -1;
410
        int64_t cooldown_replica_id = -1;
411
    };
412
413
0
    CooldownConf cooldown_conf() const {
414
0
        std::shared_lock rlock(_cooldown_conf_lock);
415
0
        return _cooldown_conf;
416
0
    }
417
418
0
    CooldownConf cooldown_conf_unlocked() const { return _cooldown_conf; }
419
420
    // Return `true` if update success
421
    bool update_cooldown_conf(int64_t cooldown_term, int64_t cooldown_replica_id);
422
423
    Status remove_all_remote_rowsets();
424
425
    void record_unused_remote_rowset(const RowsetId& rowset_id, const std::string& resource,
426
                                     int64_t num_segments);
427
428
    uint32_t calc_cold_data_compaction_score() const;
429
430
22
    std::mutex& get_cold_compaction_lock() { return _cold_compaction_lock; }
431
432
0
    std::shared_mutex& get_cooldown_conf_lock() { return _cooldown_conf_lock; }
433
434
    static void async_write_cooldown_meta(TabletSharedPtr tablet);
435
    // Return `ABORTED` if should not to retry again
436
    Status write_cooldown_meta();
437
    ////////////////////////////////////////////////////////////////////////////
438
    // end cooldown functions
439
    ////////////////////////////////////////////////////////////////////////////
440
441
    CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override;
442
    Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
443
                              DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
444
                              const RowsetIdUnorderedSet& cur_rowset_ids, int64_t lock_id = -1,
445
                              int64_t next_visible_version = -1) override;
446
447
    void merge_delete_bitmap(const DeleteBitmap& delete_bitmap);
448
    bool check_all_rowset_segment();
449
450
    void update_max_version_schema(const TabletSchemaSPtr& tablet_schema);
451
452
    void set_skip_compaction(bool skip,
453
                             CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION,
454
                             int64_t start = -1);
455
    bool should_skip_compaction(CompactionType compaction_type, int64_t now);
456
457
    std::vector<std::string> get_binlog_filepath(std::string_view binlog_version) const;
458
    std::pair<std::string, int64_t> get_binlog_info(std::string_view binlog_version) const;
459
    std::string get_rowset_binlog_meta(std::string_view binlog_version,
460
                                       std::string_view rowset_id) const;
461
    Status get_rowset_binlog_metas(const std::vector<int64_t>& binlog_versions,
462
                                   RowsetBinlogMetasPB* metas_pb);
463
    Status get_rowset_binlog_metas(Version binlog_versions, RowsetBinlogMetasPB* metas_pb);
464
    std::string get_segment_filepath(std::string_view rowset_id,
465
                                     std::string_view segment_index) const;
466
    std::string get_segment_filepath(std::string_view rowset_id, int64_t segment_index) const;
467
    bool can_add_binlog(uint64_t total_binlog_size) const;
468
    void gc_binlogs(int64_t version);
469
    Status ingest_binlog_metas(RowsetBinlogMetasPB* metas_pb);
470
471
    void report_error(const Status& st);
472
473
0
    inline int64_t get_io_error_times() const { return _io_error_times; }
474
475
0
    inline bool is_io_error_too_times() const {
476
0
        return config::max_tablet_io_errors > 0 && _io_error_times >= config::max_tablet_io_errors;
477
0
    }
478
479
21
    int64_t get_table_id() { return _tablet_meta->table_id(); }
480
481
    // binlog related functions
482
15
    bool enable_binlog() const {
483
15
        return config::enable_feature_binlog && _tablet_meta->binlog_config().is_enable();
484
15
    }
485
15
    bool enable_ccr_binlog() const {
486
15
        return enable_binlog() && _tablet_meta->binlog_config().isCCRBinlogFormat();
487
15
    }
488
334
    bool enable_row_binlog() const {
489
334
        return _tablet_meta->binlog_config().is_enable() &&
490
334
               _tablet_meta->binlog_config().isRowBinlogFormat();
491
334
    }
492
493
0
    int64_t binlog_ttl_ms() const { return _tablet_meta->binlog_config().ttl_seconds(); }
494
0
    int64_t binlog_max_bytes() const { return _tablet_meta->binlog_config().max_bytes(); }
495
496
    void set_binlog_config(BinlogConfig binlog_config);
497
498
    // row_binlog
499
2
    int32_t row_binlog_schema_hash() const { return _tablet_meta->row_binlog_schema_hash(); }
500
2
    TabletSchemaSPtr row_binlog_tablet_schema() {
501
2
        std::shared_lock rdlock(_meta_lock);
502
2
        return _tablet_meta->row_binlog_schema();
503
2
    }
504
505
0
    void set_is_full_compaction_running(bool is_full_compaction_running) {
506
0
        _is_full_compaction_running = is_full_compaction_running;
507
0
    }
508
0
    inline bool is_full_compaction_running() const { return _is_full_compaction_running; }
509
    void clear_cache() override;
510
511
3
    int32_t get_compaction_score() const { return _compaction_score; }
512
513
0
    void set_compaction_score(int32_t compaction_score) { _compaction_score = compaction_score; }
514
515
10.9k
    void add_compaction_score(int32_t score) {
516
10.9k
        if (_compaction_score < 0) {
517
10.9k
            return;
518
10.9k
        }
519
21
        _compaction_score += score;
520
21
    }
521
522
0
    void minus_compaction_score(int32_t score) {
523
0
        if (_compaction_score < 0) {
524
0
            return;
525
0
        }
526
0
        _compaction_score -= score;
527
0
    }
528
529
    Status prepare_txn(TPartitionId partition_id, TTransactionId transaction_id,
530
                       const PUniqueId& load_id, bool ingest);
531
    // TODO: commit_txn
532
533
private:
534
    Status _init_once_action();
535
    bool _contains_rowset(const RowsetId rowset_id);
536
    Status _contains_version(const Version& version);
537
538
    // Returns:
539
    // version: the max continuous version from beginning
540
    // max_version: the max version of this tablet
541
    void _max_continuous_version_from_beginning_unlocked(Version* version, Version* max_version,
542
                                                         bool* has_version_cross) const;
543
    RowsetSharedPtr _rowset_with_largest_size();
544
    /// Delete stale rowset by version. This method not only delete the version in expired rowset map,
545
    /// but also delete the version in rowset meta vector.
546
    void _delete_stale_rowset_by_version(const Version& version);
547
548
    uint32_t _calc_cumulative_compaction_score(
549
            std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy);
550
    uint32_t _calc_base_compaction_score() const;
551
552
    std::vector<RowsetSharedPtr> _pick_visible_rowsets_to_compaction(int64_t min_start_version,
553
                                                                     int64_t max_start_version);
554
555
    void _init_context_common_fields(RowsetWriterContext& context);
556
557
    ////////////////////////////////////////////////////////////////////////////
558
    // begin cooldown functions
559
    ////////////////////////////////////////////////////////////////////////////
560
    Status _cooldown_data(RowsetSharedPtr rowset);
561
    Status _follow_cooldowned_data();
562
    Status _read_cooldown_meta(const StorageResource& storage_resource,
563
                               TabletMetaPB* tablet_meta_pb);
564
    bool _has_data_to_cooldown();
565
    int64_t _get_newest_cooldown_time(const RowsetSharedPtr& rowset);
566
    ////////////////////////////////////////////////////////////////////////////
567
    // end cooldown functions
568
    ////////////////////////////////////////////////////////////////////////////
569
570
    void _clear_cache_by_rowset(const BetaRowsetSharedPtr& rowset);
571
    void check_table_size_correctness();
572
    std::string get_segment_path(const RowsetMetaSharedPtr& rs_meta, int64_t seg_id);
573
    int64_t get_segment_file_size(const RowsetMetaSharedPtr& rs_meta);
574
    int64_t get_inverted_index_file_size(const RowsetMetaSharedPtr& rs_meta);
575
576
public:
577
    static const int64_t K_INVALID_CUMULATIVE_POINT = -1;
578
579
private:
580
    StorageEngine& _engine;
581
    DataDir* _data_dir = nullptr;
582
583
    std::string _tablet_path;
584
585
    DorisCallOnce<Status> _init_once;
586
    // meta store lock is used for prevent 2 threads do checkpoint concurrently
587
    // it will be used in econ-mode in the future
588
    std::shared_mutex _meta_store_lock;
589
    std::mutex _ingest_lock;
590
    std::mutex _base_compaction_lock;
591
    std::mutex _cumulative_compaction_lock;
592
    std::shared_timed_mutex _migration_lock;
593
    std::mutex _build_inverted_index_lock;
594
595
    // In unique key table with MoW, we should guarantee that only one
596
    // writer can update rowset and delete bitmap at the same time.
597
    // We use a separate lock rather than _meta_lock, to avoid blocking read queries
598
    // during publish_txn, which might take hundreds of milliseconds
599
    mutable std::mutex _rowset_update_lock;
600
601
    // if this tablet is broken, set to true. default is false
602
    std::atomic<bool> _is_bad;
603
    // timestamp of last cumu compaction failure
604
    std::atomic<int64_t> _last_cumu_compaction_failure_millis;
605
    // timestamp of last base compaction failure
606
    std::atomic<int64_t> _last_base_compaction_failure_millis;
607
    // timestamp of last full compaction failure
608
    std::atomic<int64_t> _last_full_compaction_failure_millis;
609
    // timestamp of last cumu compaction success
610
    std::atomic<int64_t> _last_cumu_compaction_success_millis;
611
    // timestamp of last base compaction success
612
    std::atomic<int64_t> _last_base_compaction_success_millis;
613
    // timestamp of last full compaction success
614
    std::atomic<int64_t> _last_full_compaction_success_millis;
615
    // timestamp of last cumu compaction schedule time
616
    std::atomic<int64_t> _last_cumu_compaction_schedule_millis;
617
    // timestamp of last base compaction schedule time
618
    std::atomic<int64_t> _last_base_compaction_schedule_millis;
619
    // timestamp of last full compaction schedule time
620
    std::atomic<int64_t> _last_full_compaction_schedule_millis;
621
    std::atomic<int64_t> _cumulative_point;
622
    std::atomic<int64_t> _cumulative_promotion_size;
623
    std::atomic<int32_t> _newly_created_rowset_num;
624
    std::atomic<int64_t> _last_checkpoint_time;
625
    std::string _last_cumu_compaction_status;
626
    std::string _last_base_compaction_status;
627
    std::string _last_full_compaction_status;
628
629
    // single replica compaction status
630
    std::string _last_single_compaction_failure_status;
631
    Version _last_fetched_version;
632
633
    // cumulative compaction policy
634
    std::shared_ptr<CumulativeCompactionPolicy> _cumulative_compaction_policy;
635
    std::string_view _cumulative_compaction_type;
636
637
    // use a separate thread to check all tablets paths existence
638
    std::atomic<bool> _is_tablet_path_exists;
639
640
    int64_t _last_missed_version;
641
    int64_t _last_missed_time_s;
642
643
    bool _skip_cumu_compaction = false;
644
    int64_t _skip_cumu_compaction_ts;
645
646
    bool _skip_base_compaction = false;
647
    int64_t _skip_base_compaction_ts;
648
649
    // cooldown related
650
    CooldownConf _cooldown_conf;
651
    // `_cooldown_conf_lock` is used to serialize update cooldown conf and all operations that:
652
    // 1. read cooldown conf
653
    // 2. upload rowsets to remote storage
654
    // 3. update cooldown meta id
655
    mutable std::shared_mutex _cooldown_conf_lock;
656
    // `_cold_compaction_lock` is used to serialize cold data compaction and all operations that
657
    // may delete compaction input rowsets.
658
    std::mutex _cold_compaction_lock;
659
    int64_t _last_failed_follow_cooldown_time = 0;
660
661
    int64_t _io_error_times = 0;
662
663
    // partition's visible version. it sync from fe, but not real-time.
664
    atomic_shared_ptr<const VersionWithTime> _visible_version;
665
666
    std::atomic_bool _is_full_compaction_running = false;
667
668
    int32_t _compaction_score = -1;
669
    int32_t _score_check_cnt = 0;
670
};
671
672
21
inline CumulativeCompactionPolicy* Tablet::cumulative_compaction_policy() {
673
21
    return _cumulative_compaction_policy.get();
674
21
}
675
676
287
inline bool Tablet::init_succeeded() {
677
287
    return _init_once.has_called() && _init_once.stored_result().ok();
678
287
}
679
680
264
inline bool Tablet::is_used() {
681
264
    return !_is_bad && _data_dir->is_used();
682
264
}
683
684
309
inline void Tablet::register_tablet_into_dir() {
685
309
    _data_dir->register_tablet(this);
686
309
}
687
688
259
inline void Tablet::deregister_tablet_from_dir() {
689
259
    _data_dir->deregister_tablet(this);
690
259
}
691
692
39
inline int64_t Tablet::cumulative_layer_point() const {
693
39
    return _cumulative_point;
694
39
}
695
696
366
inline void Tablet::set_cumulative_layer_point(int64_t new_point) {
697
    // cumulative point should only be reset to -1, or be increased
698
366
    CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point)
699
0
            << "Unexpected cumulative point: " << new_point
700
0
            << ", origin: " << _cumulative_point.load();
701
366
    _cumulative_point = new_point;
702
366
}
703
704
63
inline int64_t Tablet::cumulative_promotion_size() const {
705
63
    return _cumulative_promotion_size;
706
63
}
707
708
79
inline void Tablet::set_cumulative_promotion_size(int64_t new_size) {
709
79
    _cumulative_promotion_size = new_size;
710
79
}
711
712
// TODO(lingbin): Why other methods that need to get information from _tablet_meta
713
// are not locked, here needs a comment to explain.
714
8
inline size_t Tablet::tablet_footprint() {
715
8
    std::shared_lock rdlock(_meta_lock);
716
8
    return _tablet_meta->tablet_footprint();
717
8
}
718
719
0
inline size_t Tablet::tablet_local_size() {
720
0
    std::shared_lock rdlock(_meta_lock);
721
0
    return _tablet_meta->tablet_local_size();
722
0
}
723
724
0
inline size_t Tablet::tablet_remote_size() {
725
0
    std::shared_lock rdlock(_meta_lock);
726
0
    return _tablet_meta->tablet_remote_size();
727
0
}
728
729
// TODO(lingbin): Why other methods which need to get information from _tablet_meta
730
// are not locked, here needs a comment to explain.
731
19
inline size_t Tablet::num_rows() {
732
19
    std::shared_lock rdlock(_meta_lock);
733
19
    return _tablet_meta->num_rows();
734
19
}
735
736
58
inline size_t Tablet::version_count() const {
737
58
    std::shared_lock rdlock(_meta_lock);
738
58
    return _tablet_meta->version_count();
739
58
}
740
741
29
inline size_t Tablet::stale_version_count() const {
742
29
    std::shared_lock rdlock(_meta_lock);
743
29
    return _tablet_meta->stale_version_count();
744
29
}
745
746
78
inline Version Tablet::max_version() const {
747
78
    std::shared_lock rdlock(_meta_lock);
748
78
    return _tablet_meta->max_version();
749
78
}
750
751
0
inline uint64_t Tablet::segment_count() const {
752
0
    std::shared_lock rdlock(_meta_lock);
753
0
    uint64_t segment_nums = 0;
754
0
    for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
755
0
        segment_nums += rs_meta->num_segments();
756
0
    }
757
0
    return segment_nums;
758
0
}
759
760
0
inline SortType Tablet::sort_type() const {
761
0
    return _tablet_meta->tablet_schema()->sort_type();
762
0
}
763
764
0
inline size_t Tablet::sort_col_num() const {
765
0
    return _tablet_meta->tablet_schema()->sort_col_num();
766
0
}
767
768
0
inline size_t Tablet::num_columns() const {
769
0
    return _tablet_meta->tablet_schema()->num_columns();
770
0
}
771
772
0
inline size_t Tablet::num_null_columns() const {
773
0
    return _tablet_meta->tablet_schema()->num_null_columns();
774
0
}
775
776
0
inline size_t Tablet::num_short_key_columns() const {
777
0
    return _tablet_meta->tablet_schema()->num_short_key_columns();
778
0
}
779
780
0
inline size_t Tablet::num_rows_per_row_block() const {
781
0
    return _tablet_meta->tablet_schema()->num_rows_per_row_block();
782
0
}
783
784
0
inline double Tablet::bloom_filter_fpp() const {
785
0
    return _tablet_meta->tablet_schema()->bloom_filter_fpp();
786
0
}
787
788
0
inline size_t Tablet::next_unique_id() const {
789
0
    return _tablet_meta->tablet_schema()->next_column_unique_id();
790
0
}
791
792
29
inline int64_t Tablet::avg_rs_meta_serialize_size() const {
793
29
    return _tablet_meta->avg_rs_meta_serialize_size();
794
29
}
795
796
} // namespace doris