Coverage Report

Created: 2026-04-11 14:25

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/cloud/cloud_tablet.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <memory>
21
22
#include "storage/partial_update_info.h"
23
#include "storage/rowset/rowset.h"
24
#include "storage/tablet/base_tablet.h"
25
26
namespace doris {
27
28
class CloudStorageEngine;
29
30
enum class WarmUpTriggerSource : int { NONE, SYNC_ROWSET, EVENT_DRIVEN, JOB };
31
32
enum class WarmUpProgress : int { NONE, DOING, DONE };
33
34
struct WarmUpState {
35
    WarmUpTriggerSource trigger_source {WarmUpTriggerSource::NONE};
36
    WarmUpProgress progress {WarmUpProgress::NONE};
37
38
    bool operator==(const WarmUpState& other) const {
39
        return trigger_source == other.trigger_source && progress == other.progress;
40
    }
41
};
42
43
struct SyncRowsetStats {
44
    int64_t get_remote_rowsets_num {0};
45
    int64_t get_remote_rowsets_rpc_ns {0};
46
47
    int64_t get_local_delete_bitmap_rowsets_num {0};
48
    int64_t get_remote_delete_bitmap_rowsets_num {0};
49
    int64_t get_remote_delete_bitmap_key_count {0};
50
    int64_t get_remote_delete_bitmap_bytes {0};
51
    int64_t get_remote_delete_bitmap_rpc_ns {0};
52
53
    int64_t get_remote_tablet_meta_rpc_ns {0};
54
    int64_t tablet_meta_cache_hit {0};
55
    int64_t tablet_meta_cache_miss {0};
56
57
    int64_t bthread_schedule_delay_ns {0};
58
    int64_t meta_lock_wait_ns {0}; // _meta_lock (std::shared_mutex) wait across all acquisitions
59
    int64_t sync_meta_lock_wait_ns {
60
            0}; // _sync_meta_lock (bthread::Mutex) wait across all acquisitions
61
};
62
63
struct SyncOptions {
64
    bool warmup_delta_data = false;
65
    bool sync_delete_bitmap = true;
66
    bool full_sync = false;
67
    bool merge_schema = false;
68
    int64_t query_version = -1;
69
};
70
71
struct RecycledRowsets {
72
    RowsetId rowset_id;
73
    int64_t num_segments;
74
    std::vector<std::string> index_file_names;
75
};
76
77
class CloudTablet final : public BaseTablet {
78
public:
79
    CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta);
80
81
    ~CloudTablet() override;
82
83
    bool exceed_version_limit(int32_t limit) override;
84
85
    Result<std::unique_ptr<RowsetWriter>> create_rowset_writer(RowsetWriterContext& context,
86
                                                               bool vertical) override;
87
88
    Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits,
89
                              const CaptureRowsetOps& opts) override;
90
91
    [[nodiscard]] Result<std::vector<Version>> capture_consistent_versions_unlocked(
92
            const Version& version_range, const CaptureRowsetOps& options) const override;
93
94
    // Capture versions with cache preference optimization.
95
    // This method prioritizes using cached/warmed-up rowsets when building version paths,
96
    // avoiding cold data reads when possible. It uses capture_consistent_versions_prefer_cache
97
    // to find a consistent version path that prefers already warmed-up rowsets.
98
    Result<std::vector<Version>> capture_versions_prefer_cache(const Version& spec_version) const;
99
100
    // Capture versions with query freshness tolerance.
101
    // This method finds a consistent version path where all rowsets are warmed up,
102
    // but allows fallback to normal capture if there are newer rowsets that should be
103
    // visible (based on freshness tolerance) but haven't been warmed up yet.
104
    // For merge-on-write tables, uses special validation to ensure data correctness.
105
    //
106
    // IMPORTANT: The returned version may be smaller than the requested version if newer
107
    // data hasn't been warmed up yet. This can cause different tablets in the same query
108
    // to read from different versions, potentially leading to inconsistent query results.
109
    //
110
    // @param options.query_freshness_tolerance_ms: Time tolerance in milliseconds. Rowsets that
111
    //        became visible within this time range (after current_time - query_freshness_tolerance_ms)
112
    //        can be skipped if not warmed up. However, if older rowsets (before this time point)
113
    //        are not warmed up, the method will fallback to normal capture.
114
    Result<std::vector<Version>> capture_versions_with_freshness_tolerance(
115
            const Version& spec_version, const CaptureRowsetOps& options) const;
116
117
530k
    size_t tablet_footprint() override {
118
530k
        return _approximate_data_size.load(std::memory_order_relaxed);
119
530k
    }
120
121
    std::string tablet_path() const override;
122
123
    // clang-format off
124
2.37M
    int64_t fetch_add_approximate_num_rowsets (int64_t x) { return _approximate_num_rowsets .fetch_add(x, std::memory_order_relaxed); }
125
170k
    int64_t fetch_add_approximate_num_segments(int64_t x) { return _approximate_num_segments.fetch_add(x, std::memory_order_relaxed); }
126
170k
    int64_t fetch_add_approximate_num_rows    (int64_t x) { return _approximate_num_rows    .fetch_add(x, std::memory_order_relaxed); }
127
170k
    int64_t fetch_add_approximate_data_size   (int64_t x) { return _approximate_data_size   .fetch_add(x, std::memory_order_relaxed); }
128
188k
    int64_t fetch_add_approximate_cumu_num_rowsets (int64_t x) { return _approximate_cumu_num_rowsets.fetch_add(x, std::memory_order_relaxed); }
129
170k
    int64_t fetch_add_approximate_cumu_num_deltas   (int64_t x) { return _approximate_cumu_num_deltas.fetch_add(x, std::memory_order_relaxed); }
130
    // clang-format on
131
132
    // meta lock must be held when calling this function
133
    void reset_approximate_stats(int64_t num_rowsets, int64_t num_segments, int64_t num_rows,
134
                                 int64_t data_size);
135
136
    // return a json string to show the compaction status of this tablet
137
    void get_compaction_status(std::string* json_result);
138
139
    // Synchronize the rowsets from meta service.
140
    // If tablet state is not `TABLET_RUNNING`, sync tablet meta and all visible rowsets.
141
    // If `query_version` > 0 and local max_version of the tablet >= `query_version`, do nothing.
142
    // If 'need_download_data_async' is true, it means that we need to download the new version
143
    // rowsets datum async.
144
    Status sync_rowsets(const SyncOptions& options = {}, SyncRowsetStats* stats = nullptr);
145
146
    // Synchronize the tablet meta from meta service.
147
    Status sync_meta();
148
149
    // If `version_overlap` is true, function will delete rowsets with overlapped version in this tablet.
150
    // If 'warmup_delta_data' is true, download the new version rowset data in background.
151
    // MUST hold EXCLUSIVE `_meta_lock`.
152
    // If 'need_download_data_async' is true, it means that we need to download the new version
153
    // rowsets datum async.
154
    void add_rowsets(std::vector<RowsetSharedPtr> to_add, bool version_overlap,
155
                     std::unique_lock<std::shared_mutex>& meta_lock,
156
                     bool warmup_delta_data = false);
157
158
    // MUST hold EXCLUSIVE `_meta_lock`.
159
    void delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
160
                        std::unique_lock<std::shared_mutex>& meta_lock);
161
162
    // Like delete_rowsets, but also removes edges from the version graph.
163
    // Used by schema change to prevent the greedy capture algorithm from
164
    // preferring stale compaction rowsets over individual SC output rowsets.
165
    // MUST hold EXCLUSIVE `_meta_lock`.
166
    void delete_rowsets_for_schema_change(const std::vector<RowsetSharedPtr>& to_delete,
167
                                          std::unique_lock<std::shared_mutex>& meta_lock);
168
169
    // When the tablet is dropped, we need to recycle cached data:
170
    // 1. The data in file cache
171
    // 2. The memory in tablet cache
172
    void clear_cache() override;
173
174
    // Return number of deleted stale rowsets
175
    uint64_t delete_expired_stale_rowsets();
176
177
633k
    bool has_stale_rowsets() const { return !_stale_rs_version_map.empty(); }
178
179
    int64_t get_cloud_base_compaction_score() const;
180
    int64_t get_cloud_cumu_compaction_score() const;
181
182
498k
    int64_t max_version_unlocked() const override { return _max_version; }
183
545k
    int64_t base_compaction_cnt() const { return _base_compaction_cnt; }
184
546k
    int64_t cumulative_compaction_cnt() const { return _cumulative_compaction_cnt; }
185
180k
    int64_t full_compaction_cnt() const { return _full_compaction_cnt; }
186
418k
    int64_t cumulative_layer_point() const {
187
418k
        return _cumulative_point.load(std::memory_order_relaxed);
188
418k
    }
189
190
181k
    void set_base_compaction_cnt(int64_t cnt) { _base_compaction_cnt = cnt; }
191
187k
    void set_cumulative_compaction_cnt(int64_t cnt) { _cumulative_compaction_cnt = cnt; }
192
180k
    void set_full_compaction_cnt(int64_t cnt) { _full_compaction_cnt = cnt; }
193
    void set_cumulative_layer_point(int64_t new_point);
194
195
233M
    int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; }
196
598
    void set_last_cumu_compaction_failure_time(int64_t millis) {
197
598
        _last_cumu_compaction_failure_millis = millis;
198
598
    }
199
200
62.4M
    int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; }
201
7.40k
    void set_last_base_compaction_failure_time(int64_t millis) {
202
7.40k
        _last_base_compaction_failure_millis = millis;
203
7.40k
    }
204
205
0
    int64_t last_full_compaction_failure_time() { return _last_full_compaction_failure_millis; }
206
2
    void set_last_full_compaction_failure_time(int64_t millis) {
207
2
        _last_full_compaction_failure_millis = millis;
208
2
    }
209
210
7.10k
    int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; }
211
13.1k
    void set_last_cumu_compaction_success_time(int64_t millis) {
212
13.1k
        _last_cumu_compaction_success_millis = millis;
213
13.1k
    }
214
215
7.09k
    int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; }
216
7.17k
    void set_last_base_compaction_success_time(int64_t millis) {
217
7.17k
        _last_base_compaction_success_millis = millis;
218
7.17k
    }
219
220
7.05k
    int64_t last_full_compaction_success_time() { return _last_full_compaction_success_millis; }
221
7.15k
    void set_last_full_compaction_success_time(int64_t millis) {
222
7.15k
        _last_full_compaction_success_millis = millis;
223
7.15k
    }
224
225
0
    int64_t last_cumu_compaction_schedule_time() { return _last_cumu_compaction_schedule_millis; }
226
390
    void set_last_cumu_compaction_schedule_time(int64_t millis) {
227
390
        _last_cumu_compaction_schedule_millis = millis;
228
390
    }
229
230
0
    int64_t last_base_compaction_schedule_time() { return _last_base_compaction_schedule_millis; }
231
1
    void set_last_base_compaction_schedule_time(int64_t millis) {
232
1
        _last_base_compaction_schedule_millis = millis;
233
1
    }
234
235
0
    int64_t last_full_compaction_schedule_time() { return _last_full_compaction_schedule_millis; }
236
105
    void set_last_full_compaction_schedule_time(int64_t millis) {
237
105
        _last_full_compaction_schedule_millis = millis;
238
105
    }
239
240
120k
    void set_last_cumu_compaction_status(std::string status) {
241
120k
        _last_cumu_compaction_status = std::move(status);
242
120k
    }
243
244
0
    std::string get_last_cumu_compaction_status() { return _last_cumu_compaction_status; }
245
246
3.85k
    void set_last_base_compaction_status(std::string status) {
247
3.85k
        _last_base_compaction_status = std::move(status);
248
3.85k
    }
249
250
0
    std::string get_last_base_compaction_status() { return _last_base_compaction_status; }
251
252
209
    void set_last_full_compaction_status(std::string status) {
253
209
        _last_full_compaction_status = std::move(status);
254
209
    }
255
256
0
    std::string get_last_full_compaction_status() { return _last_full_compaction_status; }
257
258
156k
    int64_t alter_version() const { return _alter_version; }
259
82.1k
    void set_alter_version(int64_t alter_version) { _alter_version = alter_version; }
260
261
    // Last active cluster info for compaction read-write separation
262
296M
    std::string last_active_cluster_id() const {
263
296M
        std::shared_lock lock(_cluster_info_mutex);
264
296M
        return _last_active_cluster_id;
265
296M
    }
266
    int64_t last_active_time_ms() const {
267
        std::shared_lock lock(_cluster_info_mutex);
268
        return _last_active_time_ms;
269
    }
270
15.4k
    void set_last_active_cluster_info(const std::string& cluster_id, int64_t time_ms) {
271
15.4k
        std::unique_lock lock(_cluster_info_mutex);
272
15.4k
        _last_active_cluster_id = cluster_id;
273
15.4k
        _last_active_time_ms = time_ms;
274
15.4k
    }
275
276
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction();
277
278
880k
    inline Version max_version() const {
279
880k
        std::shared_lock rdlock(_meta_lock);
280
880k
        return _tablet_meta->max_version();
281
880k
    }
282
283
126k
    int64_t base_size() const { return _base_size; }
284
285
    std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction();
286
    Result<RowsetSharedPtr> pick_a_rowset_for_index_change(int schema_version,
287
                                                           bool& is_base_rowset);
288
    Status check_rowset_schema_for_build_index(std::vector<TColumn>& columns, int schema_version);
289
290
0
    std::mutex& get_base_compaction_lock() { return _base_compaction_lock; }
291
0
    std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; }
292
293
    Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer(
294
            const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info,
295
            int64_t txn_expiration = 0) override;
296
297
    CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override;
298
299
    Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
300
                              DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
301
                              const RowsetIdUnorderedSet& cur_rowset_ids, int64_t lock_id = -1,
302
                              int64_t next_visible_version = -1) override;
303
304
    Status save_delete_bitmap_to_ms(int64_t cur_version, int64_t txn_id,
305
                                    DeleteBitmapPtr delete_bitmap, int64_t lock_id,
306
                                    int64_t next_visible_version, RowsetSharedPtr rowset);
307
308
    Status calc_delete_bitmap_for_compaction(const std::vector<RowsetSharedPtr>& input_rowsets,
309
                                             const RowsetSharedPtr& output_rowset,
310
                                             const RowIdConversion& rowid_conversion,
311
                                             ReaderType compaction_type, int64_t merged_rows,
312
                                             int64_t filtered_rows, int64_t initiator,
313
                                             DeleteBitmapPtr& output_rowset_delete_bitmap,
314
                                             bool allow_delete_in_cumu_compaction,
315
                                             int64_t& get_delete_bitmap_lock_start_time);
316
317
    // Find the missed versions until the spec_version.
318
    //
319
    // for example:
320
    //     [0-4][5-5][8-8][9-9][14-14]
321
    // if spec_version = 12, it will return [6-7],[10-12]
322
    Versions calc_missed_versions(int64_t spec_version, Versions existing_versions) const override;
323
324
48.9k
    std::mutex& get_rowset_update_lock() { return _rowset_update_lock; }
325
326
157k
    bthread::Mutex& get_sync_meta_lock() { return _sync_meta_lock; }
327
328
128k
    const auto& rowset_map() const { return _rs_version_map; }
329
330
    int64_t last_sync_time_s = 0;
331
    int64_t last_load_time_ms = 0;
332
    int64_t last_base_compaction_success_time_ms = 0;
333
    int64_t last_cumu_compaction_success_time_ms = 0;
334
    int64_t last_cumu_no_suitable_version_ms = 0;
335
    int64_t last_access_time_ms = 0;
336
337
    std::atomic<int64_t> local_read_time_us = 0;
338
    std::atomic<int64_t> remote_read_time_us = 0;
339
    std::atomic<int64_t> exec_compaction_time_us = 0;
340
341
    void build_tablet_report_info(TTabletInfo* tablet_info);
342
343
    // check that if the delete bitmap in delete bitmap cache has the same cardinality with the expected_delete_bitmap's
344
    Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap* expected_delete_bitmap) override;
345
346
    void agg_delete_bitmap_for_compaction(int64_t start_version, int64_t end_version,
347
                                          const std::vector<RowsetSharedPtr>& pre_rowsets,
348
                                          DeleteBitmapPtr& new_delete_bitmap,
349
                                          std::map<std::string, int64_t>& pre_rowset_to_versions);
350
351
    bool need_remove_unused_rowsets();
352
353
    void add_unused_rowsets(const std::vector<RowsetSharedPtr>& rowsets);
354
    void remove_unused_rowsets();
355
356
    // For each given rowset not in active use, clears its file cache and returns its
357
    // ID, segment count, and index file names as RecycledRowsets entries.
358
    static std::vector<RecycledRowsets> recycle_cached_data(
359
            const std::vector<RowsetSharedPtr>& rowsets);
360
361
    // Add warmup state management
362
    WarmUpState get_rowset_warmup_state(RowsetId rowset_id);
363
    bool add_rowset_warmup_state(
364
            const RowsetMeta& rowset, WarmUpTriggerSource source,
365
            std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now());
366
    bool update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, RowsetId rowset_id,
367
                                                     int64_t delta);
368
    bool update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source,
369
                                                              RowsetId rowset_id, int64_t delta);
370
    WarmUpState complete_rowset_segment_warmup(WarmUpTriggerSource trigger_source,
371
                                               RowsetId rowset_id, Status status,
372
                                               int64_t segment_num, int64_t inverted_idx_num);
373
374
    bool is_rowset_warmed_up(const RowsetId& rowset_id) const;
375
376
    void add_warmed_up_rowset(const RowsetId& rowset_id);
377
    // Test helper: add a rowset to the warmup state map with DOING progress,
378
    // so that is_rowset_warmed_up() returns false for it.
379
    void add_not_warmed_up_rowset(const RowsetId& rowset_id);
380
381
    // Try to apply visible pending rowsets to tablet meta in version order
382
    // This should be called after receiving FE notification or when new rowsets are added
383
    // @return Status::OK() if successfully applied, error otherwise
384
    void apply_visible_pending_rowsets();
385
386
    void try_make_committed_rs_visible(int64_t txn_id, int64_t visible_version,
387
                                       int64_t version_update_time_ms);
388
    void try_make_committed_rs_visible_for_mow(int64_t txn_id, int64_t visible_version,
389
                                               int64_t version_update_time_ms);
390
391
    void clear_unused_visible_pending_rowsets();
392
393
5
    std::string rowset_warmup_digest() const {
394
5
        std::string res;
395
104
        auto add_log = [&](const RowsetSharedPtr& rs) {
396
104
            auto tmp = fmt::format("{}{}", rs->rowset_id().to_string(), rs->version().to_string());
397
104
            if (_rowset_warm_up_states.contains(rs->rowset_id())) {
398
104
                tmp += fmt::format(
399
104
                        ", progress={}, segments_warmed_up={}/{}, inverted_idx_warmed_up={}/{}",
400
104
                        _rowset_warm_up_states.at(rs->rowset_id()).state.progress,
401
104
                        _rowset_warm_up_states.at(rs->rowset_id()).num_segments_warmed_up,
402
104
                        _rowset_warm_up_states.at(rs->rowset_id()).num_segments,
403
104
                        _rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx_warmed_up,
404
104
                        _rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx);
405
104
            }
406
104
            res += fmt::format("[{}],", tmp);
407
104
        };
408
5
        traverse_rowsets_unlocked(add_log, true);
409
5
        return res;
410
5
    }
411
412
private:
413
    // FIXME(plat1ko): No need to record base size if rowsets are ordered by version
414
    void update_base_size(const Rowset& rs);
415
416
    Status sync_if_not_running(SyncRowsetStats* stats = nullptr);
417
418
    bool add_rowset_warmup_state_unlocked(
419
            const RowsetMeta& rowset, WarmUpTriggerSource source,
420
            std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now());
421
422
    // used by capture_rs_reader_xxx functions
423
    bool rowset_is_warmed_up_unlocked(int64_t start_version, int64_t end_version) const;
424
425
    // Check if a rowset should be visible but not warmed up within freshness tolerance
426
    bool _check_rowset_should_be_visible_but_not_warmed_up(
427
            const RowsetMetaSharedPtr& rs_meta, int64_t path_max_version,
428
            std::chrono::system_clock::time_point freshness_limit_tp) const;
429
430
    // Submit a segment download task for warming up
431
    void _submit_segment_download_task(const RowsetSharedPtr& rs,
432
                                       const StorageResource* storage_resource, int seg_id,
433
                                       int64_t expiration_time);
434
435
    // Submit an inverted index download task for warming up
436
    void _submit_inverted_index_download_task(const RowsetSharedPtr& rs,
437
                                              const StorageResource* storage_resource,
438
                                              const io::Path& idx_path, int64_t idx_size,
439
                                              int64_t expiration_time);
440
441
    // Add rowsets directly with warmup
442
    void _add_rowsets_directly(std::vector<RowsetSharedPtr>& rowsets, bool warmup_delta_data);
443
444
    CloudStorageEngine& _engine;
445
446
    // this mutex MUST ONLY be used when sync meta
447
    bthread::Mutex _sync_meta_lock;
448
    // ATTENTION: lock order should be: _sync_meta_lock -> _meta_lock
449
450
    std::atomic<int64_t> _cumulative_point {-1};
451
    std::atomic<int64_t> _approximate_num_rowsets {-1};
452
    std::atomic<int64_t> _approximate_num_segments {-1};
453
    std::atomic<int64_t> _approximate_num_rows {-1};
454
    std::atomic<int64_t> _approximate_data_size {-1};
455
    std::atomic<int64_t> _approximate_cumu_num_rowsets {-1};
456
    // Number of sorted arrays (e.g. for rowset with N segments, if rowset is overlapping, delta is N, otherwise 1) after cumu point
457
    std::atomic<int64_t> _approximate_cumu_num_deltas {-1};
458
459
    // timestamp of last cumu compaction failure
460
    std::atomic<int64_t> _last_cumu_compaction_failure_millis;
461
    // timestamp of last base compaction failure
462
    std::atomic<int64_t> _last_base_compaction_failure_millis;
463
    // timestamp of last full compaction failure
464
    std::atomic<int64_t> _last_full_compaction_failure_millis;
465
    // timestamp of last cumu compaction success
466
    std::atomic<int64_t> _last_cumu_compaction_success_millis;
467
    // timestamp of last base compaction success
468
    std::atomic<int64_t> _last_base_compaction_success_millis;
469
    // timestamp of last full compaction success
470
    std::atomic<int64_t> _last_full_compaction_success_millis;
471
    // timestamp of last cumu compaction schedule time
472
    std::atomic<int64_t> _last_cumu_compaction_schedule_millis;
473
    // timestamp of last base compaction schedule time
474
    std::atomic<int64_t> _last_base_compaction_schedule_millis;
475
    // timestamp of last full compaction schedule time
476
    std::atomic<int64_t> _last_full_compaction_schedule_millis;
477
478
    std::string _last_cumu_compaction_status;
479
    std::string _last_base_compaction_status;
480
    std::string _last_full_compaction_status;
481
482
    int64_t _base_compaction_cnt = 0;
483
    int64_t _cumulative_compaction_cnt = 0;
484
    int64_t _full_compaction_cnt = 0;
485
    int64_t _max_version = -1;
486
    int64_t _base_size = 0;
487
    int64_t _alter_version = -1;
488
489
    std::mutex _base_compaction_lock;
490
    std::mutex _cumulative_compaction_lock;
491
492
    // To avoid multiple calc delete bitmap tasks on same (txn_id, tablet_id) with different
493
    // signatures being executed concurrently, we use _rowset_update_lock to serialize them
494
    mutable std::mutex _rowset_update_lock;
495
496
    // unused_rowsets, [start_version, end_version]
497
    std::mutex _gc_mutex;
498
    std::unordered_map<RowsetId, RowsetSharedPtr> _unused_rowsets;
499
    std::vector<std::pair<std::vector<RowsetId>, DeleteBitmapKeyRanges>> _unused_delete_bitmap;
500
501
    // for warm up states management
502
    struct RowsetWarmUpInfo {
503
        WarmUpState state;
504
        int64_t num_segments = 0;
505
        int64_t num_inverted_idx = 0;
506
        int64_t num_segments_warmed_up = 0;
507
        int64_t num_inverted_idx_warmed_up = 0;
508
        std::chrono::steady_clock::time_point start_tp;
509
510
55.9k
        void done(int64_t input_num_segments, int64_t input_num_inverted_idx) {
511
55.9k
            num_segments_warmed_up += input_num_segments;
512
55.9k
            num_inverted_idx_warmed_up += input_num_inverted_idx;
513
55.9k
            update_state();
514
55.9k
        }
515
516
55.9k
        bool has_finished() const {
517
55.9k
            return (num_segments_warmed_up >= num_segments) &&
518
55.9k
                   (num_inverted_idx_warmed_up >= num_inverted_idx);
519
55.9k
        }
520
521
        void update_state();
522
    };
523
    std::unordered_map<RowsetId, RowsetWarmUpInfo> _rowset_warm_up_states;
524
525
    mutable std::shared_mutex _warmed_up_rowsets_mutex;
526
    std::unordered_set<RowsetId> _warmed_up_rowsets;
527
528
    // Cluster info for compaction read-write separation
529
    mutable std::shared_mutex _cluster_info_mutex;
530
    std::string _last_active_cluster_id;
531
    int64_t _last_active_time_ms {0};
532
533
    // Map: version -> <rowset_meta, expiration_time>
534
    // Stores rowsets that have been notified by FE but not yet added to tablet meta
535
    // due to out-of-order notification or version discontinuity
536
    struct VisiblePendingRowset {
537
        const bool is_empty_rowset;
538
        const int64_t expiration_time; // seconds since epoch
539
        RowsetMetaSharedPtr rowset_meta;
540
541
        VisiblePendingRowset(RowsetMetaSharedPtr rowset_meta_, int64_t expiration_time_,
542
                             bool is_empty_rowset_ = false)
543
119k
                : is_empty_rowset(is_empty_rowset_),
544
119k
                  expiration_time(expiration_time_),
545
119k
                  rowset_meta(std::move(rowset_meta_)) {}
546
    };
547
    mutable std::mutex _visible_pending_rs_lock;
548
    std::map<int64_t, VisiblePendingRowset> _visible_pending_rs_map;
549
};
550
551
using CloudTabletSPtr = std::shared_ptr<CloudTablet>;
552
553
} // namespace doris