Coverage Report

Created: 2026-09-17 13:03

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/rowset/rowset_meta.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
19
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
20
21
#include <gen_cpp/olap_file.pb.h>
22
#include <glog/logging.h>
23
24
#include <atomic>
25
#include <chrono>
26
#include <cstddef>
27
#include <cstdint>
28
#include <iterator>
29
#include <memory>
30
#include <string>
31
#include <vector>
32
33
#include "common/cast_set.h"
34
#include "common/check.h"
35
#include "common/config.h"
36
#include "common/status.h"
37
#include "io/fs/encrypted_fs_factory.h"
38
#include "io/fs/file_system.h"
39
#include "runtime/memory/lru_cache_policy.h"
40
#include "storage/metadata_adder.h"
41
#include "storage/olap_common.h"
42
#include "storage/rowset/rowset_fwd.h"
43
#include "storage/rowset/rowset_segment_id.h"
44
#include "storage/storage_policy.h"
45
#include "storage/tablet/tablet_fwd.h"
46
#include "util/once.h"
47
48
namespace doris {
49
50
class RowsetSegmentMetaView;
51
class RowsetSegmentMetaRange;
52
53
class RowsetMeta : public MetadataAdder<RowsetMeta> {
54
public:
55
1.10M
    RowsetMeta() = default;
56
    ~RowsetMeta();
57
58
    bool init(std::string_view pb_rowset_meta);
59
60
    bool init(const RowsetMeta* rowset_meta);
61
62
    bool init_from_pb(const RowsetMetaPB& rowset_meta_pb);
63
64
    bool init_from_json(const std::string& json_rowset_meta);
65
66
6
    bool serialize(std::string* value) { return _serialize_to_pb(value); }
67
68
    bool json_rowset_meta(std::string* json_rowset_meta);
69
70
    // If the rowset is a local rowset, return the global local file system.
71
    // Otherwise, return the remote file system corresponding to rowset's resource id.
72
    // Note that if the resource id cannot be found for the corresponding remote file system, nullptr will be returned.
73
    MOCK_FUNCTION io::FileSystemSPtr fs();
74
75
    // The bare file system holding this rowset's files. It resolves neither packed files nor
76
    // encryption, so it cannot open a segment whose bytes live inside a packed object. Prefer
77
    // `fs()`, or `packed_physical_fs()` when the raw bytes are what you are after.
78
    io::FileSystemSPtr physical_fs();
79
80
    // Same as `physical_fs()`, but additionally wrapped with `PackedFileSystem` when this
81
    // rowset's files are packed into shared objects, so that segment/index paths still
82
    // resolve. Unlike `fs()`, no encryption layer is applied, i.e. reads return the raw
83
    // on-disk bytes. Callers that inspect the physical layout of a file (encryption footer,
84
    // magic code, ...) must use this instead of `physical_fs()`, otherwise packed files
85
    // cannot be opened at all.
86
    io::FileSystemSPtr packed_physical_fs();
87
88
    Result<const StorageResource*> remote_storage_resource();
89
90
    void set_remote_storage_resource(StorageResource resource);
91
92
1.74M
    const std::string& resource_id() const { return _rowset_meta_pb.resource_id(); }
93
94
1.25k
    void set_resource_id(const std::string& resource_id) {
95
1.25k
        _rowset_meta_pb.set_resource_id(resource_id);
96
1.25k
    }
97
98
47.8M
    bool is_local() const { return !_rowset_meta_pb.has_resource_id(); }
99
100
    bool has_variant_type_in_schema() const;
101
102
25.0M
    RowsetId rowset_id() const { return _rowset_id; }
103
104
212k
    void set_rowset_id(const RowsetId& rowset_id) {
105
        // rowset id is a required field, just set it to 0
106
212k
        _rowset_meta_pb.set_rowset_id(0);
107
212k
        _rowset_id = rowset_id;
108
212k
        _rowset_meta_pb.set_rowset_id_v2(rowset_id.to_string());
109
212k
    }
110
111
8.69M
    int64_t tablet_id() const { return _rowset_meta_pb.tablet_id(); }
112
113
212k
    void set_tablet_id(int64_t tablet_id) { _rowset_meta_pb.set_tablet_id(tablet_id); }
114
115
5.11k
    int64_t db_id() const { return _rowset_meta_pb.db_id(); }
116
117
206k
    void set_db_id(int64_t db_id) { _rowset_meta_pb.set_db_id(db_id); }
118
119
4.93k
    int64_t table_id() const { return _rowset_meta_pb.table_id(); }
120
121
207k
    void set_table_id(int64_t table_id) { _rowset_meta_pb.set_table_id(table_id); }
122
123
    int64_t index_id() const { return _rowset_meta_pb.index_id(); }
124
125
206k
    void set_index_id(int64_t index_id) { _rowset_meta_pb.set_index_id(index_id); }
126
127
    bool has_inverted_index_storage_format() const {
128
        return _rowset_meta_pb.has_inverted_index_storage_format();
129
    }
130
131
    InvertedIndexStorageFormatPB inverted_index_storage_format() const {
132
        return _rowset_meta_pb.inverted_index_storage_format();
133
    }
134
135
    void set_inverted_index_storage_format(InvertedIndexStorageFormatPB format);
136
137
12.9k
    TabletUid tablet_uid() const { return _rowset_meta_pb.tablet_uid(); }
138
139
8.66k
    void set_tablet_uid(TabletUid tablet_uid) {
140
8.66k
        *(_rowset_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto();
141
8.66k
    }
142
143
680k
    int64_t txn_id() const { return _rowset_meta_pb.txn_id(); }
144
145
204k
    void set_txn_id(int64_t txn_id) { _rowset_meta_pb.set_txn_id(txn_id); }
146
147
1.41k
    int32_t tablet_schema_hash() const { return _rowset_meta_pb.tablet_schema_hash(); }
148
149
207k
    void set_tablet_schema_hash(int32_t tablet_schema_hash) {
150
207k
        _rowset_meta_pb.set_tablet_schema_hash(tablet_schema_hash);
151
207k
    }
152
153
2.52k
    void mark_row_binlog() { _rowset_meta_pb.set_is_row_binlog(true); }
154
155
155k
    bool is_row_binlog() const {
156
155k
        return _rowset_meta_pb.has_is_row_binlog() && _rowset_meta_pb.is_row_binlog();
157
155k
    }
158
159
2.00M
    RowsetTypePB rowset_type() const { return _rowset_meta_pb.rowset_type(); }
160
161
210k
    void set_rowset_type(RowsetTypePB rowset_type) { _rowset_meta_pb.set_rowset_type(rowset_type); }
162
163
590k
    RowsetStatePB rowset_state() const { return _rowset_meta_pb.rowset_state(); }
164
165
419k
    void set_rowset_state(RowsetStatePB rowset_state) {
166
419k
        _rowset_meta_pb.set_rowset_state(rowset_state);
167
419k
    }
168
169
49.6M
    Version version() const {
170
49.6M
        return {_rowset_meta_pb.start_version(), _rowset_meta_pb.end_version()};
171
49.6M
    }
172
173
259k
    void set_version(Version version) {
174
259k
        _rowset_meta_pb.set_start_version(version.first);
175
259k
        _rowset_meta_pb.set_end_version(version.second);
176
259k
    }
177
178
1.13M
    bool has_version() const {
179
1.13M
        return _rowset_meta_pb.has_start_version() && _rowset_meta_pb.has_end_version();
180
1.13M
    }
181
182
4.95M
    int64_t start_version() const { return _rowset_meta_pb.start_version(); }
183
184
7.87M
    int64_t end_version() const { return _rowset_meta_pb.end_version(); }
185
186
9.07M
    int64_t num_rows() const { return _rowset_meta_pb.num_rows(); }
187
188
238k
    void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); }
189
190
231k
    void set_num_segment_rows(const std::vector<uint32_t>& num_segment_rows) {
191
231k
        _rowset_meta_pb.mutable_num_segment_rows()->Assign(num_segment_rows.cbegin(),
192
231k
                                                           num_segment_rows.cend());
193
231k
    }
194
195
137
    void get_num_segment_rows(std::vector<uint32_t>* num_segment_rows) const {
196
137
        num_segment_rows->assign(_rowset_meta_pb.num_segment_rows().cbegin(),
197
137
                                 _rowset_meta_pb.num_segment_rows().cend());
198
137
    }
199
200
169k
    auto& get_num_segment_rows() const { return _rowset_meta_pb.num_segment_rows(); }
201
202
14
    void set_segment_group_sizes(const std::vector<int32_t>& segment_group_sizes) {
203
14
        DORIS_CHECK_GT(segment_group_sizes.size(), 1);
204
14
        int64_t segment_count = 0;
205
45
        for (const auto group_size : segment_group_sizes) {
206
45
            DORIS_CHECK_GT(group_size, 0);
207
45
            segment_count += group_size;
208
45
        }
209
14
        DORIS_CHECK_EQ(segment_count, num_segments());
210
14
        _rowset_meta_pb.mutable_segment_group_sizes()->Assign(segment_group_sizes.cbegin(),
211
14
                                                              segment_group_sizes.cend());
212
14
    }
213
214
    void clear_segment_group_sizes() { _rowset_meta_pb.clear_segment_group_sizes(); }
215
216
34
    const auto& segment_group_sizes() const { return _rowset_meta_pb.segment_group_sizes(); }
217
218
10.4M
    int64_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); }
219
220
250k
    void set_total_disk_size(int64_t total_disk_size) {
221
250k
        _rowset_meta_pb.set_total_disk_size(total_disk_size);
222
250k
    }
223
224
9.04M
    int64_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); }
225
226
238k
    void set_data_disk_size(int64_t data_disk_size) {
227
238k
        _rowset_meta_pb.set_data_disk_size(data_disk_size);
228
238k
    }
229
230
2.52M
    int64_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); }
231
232
238k
    void set_index_disk_size(int64_t index_disk_size) {
233
238k
        _rowset_meta_pb.set_index_disk_size(index_disk_size);
234
238k
    }
235
236
0
    void zone_maps(std::vector<::doris::ZoneMap>* zone_maps) {
237
0
        for (const ::doris::ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) {
238
0
            zone_maps->push_back(zone_map);
239
0
        }
240
0
    }
241
242
0
    void set_zone_maps(const std::vector<::doris::ZoneMap>& zone_maps) {
243
0
        for (const ::doris::ZoneMap& zone_map : zone_maps) {
244
0
            ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps();
245
0
            *new_zone_map = zone_map;
246
0
        }
247
0
    }
248
249
0
    void add_zone_map(const ::doris::ZoneMap& zone_map) {
250
0
        ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps();
251
0
        *new_zone_map = zone_map;
252
0
    }
253
254
5.12M
    bool has_delete_predicate() const { return _rowset_meta_pb.has_delete_predicate(); }
255
256
8.99k
    const DeletePredicatePB& delete_predicate() const { return _rowset_meta_pb.delete_predicate(); }
257
258
0
    DeletePredicatePB* mutable_delete_predicate() {
259
0
        return _rowset_meta_pb.mutable_delete_predicate();
260
0
    }
261
262
3.24k
    void set_delete_predicate(DeletePredicatePB delete_predicate) {
263
3.24k
        DeletePredicatePB* new_delete_condition = _rowset_meta_pb.mutable_delete_predicate();
264
3.24k
        *new_delete_condition = std::move(delete_predicate);
265
3.24k
    }
266
267
8.29k
    bool empty() const { return _rowset_meta_pb.empty(); }
268
269
232k
    void set_empty(bool empty) { _rowset_meta_pb.set_empty(empty); }
270
271
136
    PUniqueId load_id() const { return _rowset_meta_pb.load_id(); }
272
273
187k
    void set_load_id(PUniqueId load_id) {
274
187k
        PUniqueId* new_load_id = _rowset_meta_pb.mutable_load_id();
275
187k
        new_load_id->set_hi(load_id.hi());
276
187k
        new_load_id->set_lo(load_id.lo());
277
187k
    }
278
279
206k
    void set_job_id(const std::string& job_id) { _rowset_meta_pb.set_job_id(job_id); }
280
281
0
    const std::string& job_id() const { return _rowset_meta_pb.job_id(); }
282
283
0
    bool delete_flag() const { return _rowset_meta_pb.delete_flag(); }
284
285
130k
    int64_t creation_time() const { return _rowset_meta_pb.creation_time(); }
286
287
233k
    void set_creation_time(int64_t creation_time) {
288
233k
        return _rowset_meta_pb.set_creation_time(creation_time);
289
233k
    }
290
291
528k
    int64_t stale_at() const {
292
528k
        int64_t stale_time = _stale_at_s.load();
293
528k
        return stale_time > 0 ? stale_time : _rowset_meta_pb.creation_time();
294
528k
    }
295
296
12.4k
    bool has_stale_at() const { return _stale_at_s.load() > 0; }
297
298
89.8k
    void set_stale_at(int64_t stale_at) { _stale_at_s.store(stale_at); }
299
300
11.5k
    int64_t partition_id() const { return _rowset_meta_pb.partition_id(); }
301
302
207k
    void set_partition_id(int64_t partition_id) {
303
207k
        return _rowset_meta_pb.set_partition_id(partition_id);
304
207k
    }
305
306
31.6M
    int64_t num_segments() const {
307
31.6M
        DCHECK(_rowset_meta_pb.segment_ids_size() == 0 ||
308
31.6M
               _rowset_meta_pb.segment_ids_size() == _rowset_meta_pb.num_segments());
309
31.6M
        return _rowset_meta_pb.num_segments();
310
31.6M
    }
311
312
274k
    void set_num_segments(int64_t num_segments) { _rowset_meta_pb.set_num_segments(num_segments); }
313
314
4.81M
    bool has_segment_ids() const { return _rowset_meta_pb.segment_ids_size() > 0; }
315
316
291k
    const auto& segment_ids() const { return _rowset_meta_pb.segment_ids(); }
317
318
    void set_segment_ids(const std::vector<int64_t>& segment_ids);
319
320
3.85M
    int64_t segment_id(size_t pos) const {
321
3.85M
        DORIS_CHECK_LT(pos, cast_set<size_t>(num_segments()));
322
3.85M
        return has_segment_ids() ? _rowset_meta_pb.segment_ids(cast_set<int>(pos))
323
3.85M
                                 : cast_set<int64_t>(pos);
324
3.85M
    }
325
326
3.85M
    RowsetSegmentRef segment_ref(size_t pos) const { return {pos, segment_id(pos)}; }
327
328
    RowsetSegmentMetaView segment(size_t pos) const;
329
330
    RowsetSegmentMetaRange segments() const;
331
332
    size_t position_of(int64_t seg_id) const;
333
334
    // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta.
335
    void to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema = false) const;
336
337
    // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta.
338
    RowsetMetaPB get_rowset_pb(bool skip_schema = false) const;
339
340
70
    inline DeletePredicatePB* mutable_delete_pred_pb() {
341
70
        return _rowset_meta_pb.mutable_delete_predicate();
342
70
    }
343
344
123k
    bool is_singleton_delta() const {
345
123k
        return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version();
346
123k
    }
347
348
    // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta.
349
    // But, this check behavior may cost a lot of time when it is frequent.
350
    // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true,
351
    // And next time when we want to check if this rowset is in rowset mata manager's meta, we can
352
    // check is_remove_from_rowset_meta() first.
353
4.07k
    void set_remove_from_rowset_meta() { _is_removed_from_rowset_meta = true; }
354
355
4.07k
    bool is_remove_from_rowset_meta() const { return _is_removed_from_rowset_meta; }
356
357
34.0k
    SegmentsOverlapPB segments_overlap() const { return _rowset_meta_pb.segments_overlap_pb(); }
358
359
298k
    void set_segments_overlap(SegmentsOverlapPB segments_overlap) {
360
298k
        _rowset_meta_pb.set_segments_overlap_pb(segments_overlap);
361
298k
    }
362
363
621k
    static bool comparator(const RowsetMetaSharedPtr& left, const RowsetMetaSharedPtr& right) {
364
621k
        return left->end_version() < right->end_version();
365
621k
    }
366
367
    // return true if segments in this rowset has overlapping data.
368
    // this is not same as `segments_overlap()` method.
369
    // `segments_overlap()` only return the value of "segments_overlap" field in rowset meta,
370
    // but "segments_overlap" may be UNKNOWN.
371
    //
372
    // Returns true if all of the following conditions are met:
373
    // 1. The rowset contains more than one segment.
374
    // 2. segments_overlap() is not NONOVERLAPPING (OVERLAP_UNKNOWN, OVERLAPPING, and
375
    //    NONOVERLAPPING_WITHIN_GROUP are considered overlapping).
376
    // 3. The rowset has a singleton version, except row-binlog LMax quick merge rowsets that
377
    //    explicitly set segments_overlap() to OVERLAPPING.
378
5.12M
    bool is_segments_overlapping() const {
379
5.12M
        return num_segments() > 1 && segments_overlap() != NONOVERLAPPING &&
380
5.12M
               (is_singleton_delta() || (is_row_binlog() && segments_overlap() == OVERLAPPING));
381
5.12M
    }
382
383
52
    bool produced_by_compaction() const {
384
52
        return has_version() && (start_version() < end_version() ||
385
51
                                 (start_version() == end_version() &&
386
1
                                  (segments_overlap() == NONOVERLAPPING ||
387
1
                                   segments_overlap() == NONOVERLAPPING_WITHIN_GROUP)));
388
52
    }
389
390
    // get the compaction score of this rowset.
391
    // if segments are overlapping, the score equals to the number of segments,
392
    // otherwise, score is 1.
393
2.29M
    uint32_t get_compaction_score() const {
394
2.29M
        uint32_t score = 0;
395
2.29M
        if (!is_segments_overlapping()) {
396
2.28M
            score = 1;
397
2.28M
        } else {
398
3.31k
            auto num_seg = num_segments();
399
3.31k
            DCHECK_GT(num_seg, 0);
400
3.31k
            score = cast_set<uint32_t>(num_seg);
401
3.31k
            CHECK(score > 0);
402
3.31k
        }
403
2.29M
        return score;
404
2.29M
    }
405
406
76.9k
    uint32_t get_merge_way_num() const {
407
76.9k
        uint32_t way_num = 0;
408
77.3k
        if (!is_segments_overlapping()) {
409
77.3k
            if (num_segments() == 0) {
410
50.2k
                way_num = 0;
411
50.2k
            } else {
412
27.0k
                way_num = 1;
413
27.0k
            }
414
18.4E
        } else {
415
18.4E
            auto num_seg = num_segments();
416
18.4E
            DCHECK_GT(num_seg, 0);
417
418
18.4E
            way_num = cast_set<uint32_t>(num_seg);
419
18.4E
            CHECK(way_num > 0);
420
18.4E
        }
421
76.9k
        return way_num;
422
76.9k
    }
423
424
260
    void get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) const {
425
275
        for (const KeyBoundsPB& key_range : _rowset_meta_pb.segments_key_bounds()) {
426
275
            segments_key_bounds->push_back(key_range);
427
275
        }
428
260
    }
429
430
4.19M
    auto& get_segments_key_bounds() const { return _rowset_meta_pb.segments_key_bounds(); }
431
432
5.28M
    bool is_segments_key_bounds_truncated() const {
433
5.28M
        return _rowset_meta_pb.has_segments_key_bounds_truncated() &&
434
5.28M
               _rowset_meta_pb.segments_key_bounds_truncated();
435
5.28M
    }
436
437
237k
    void set_segments_key_bounds_truncated(bool truncated) {
438
237k
        _rowset_meta_pb.set_segments_key_bounds_truncated(truncated);
439
237k
    }
440
441
    // When true, `segments_key_bounds` holds a single aggregated
442
    // [rowset_min, rowset_max] entry instead of per-segment bounds.
443
4.17M
    bool is_segments_key_bounds_aggregated() const {
444
4.17M
        return _rowset_meta_pb.has_segments_key_bounds_aggregated() &&
445
4.17M
               _rowset_meta_pb.segments_key_bounds_aggregated();
446
4.17M
    }
447
448
231k
    void set_segments_key_bounds_aggregated(bool aggregated) {
449
231k
        _rowset_meta_pb.set_segments_key_bounds_aggregated(aggregated);
450
231k
    }
451
452
1.07M
    bool get_first_segment_key_bound(KeyBoundsPB* key_bounds) {
453
        // for compatibility, old version has not segment key bounds
454
1.07M
        if (_rowset_meta_pb.segments_key_bounds_size() == 0) {
455
0
            return false;
456
0
        }
457
1.07M
        *key_bounds = *_rowset_meta_pb.segments_key_bounds().begin();
458
1.07M
        return true;
459
1.07M
    }
460
461
814k
    bool get_last_segment_key_bound(KeyBoundsPB* key_bounds) {
462
814k
        if (_rowset_meta_pb.segments_key_bounds_size() == 0) {
463
0
            return false;
464
0
        }
465
814k
        *key_bounds = *_rowset_meta_pb.segments_key_bounds().rbegin();
466
814k
        return true;
467
814k
    }
468
469
    // If `aggregate_into_single` is true, collapse per-segment bounds into a single
470
    // [rowset_min, rowset_max] entry and mark this rowset as aggregated.
471
    void set_segments_key_bounds(const std::vector<KeyBoundsPB>& segments_key_bounds,
472
                                 bool aggregate_into_single = false);
473
474
34
    void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) {
475
34
        *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds);
476
34
        set_segments_overlap(OVERLAPPING);
477
34
    }
478
479
211k
    void set_newest_write_timestamp(int64_t timestamp) {
480
211k
        _rowset_meta_pb.set_newest_write_timestamp(timestamp);
481
211k
    }
482
483
641k
    int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); }
484
485
    // for cloud only
486
290
    bool has_visible_ts_ms() const { return _rowset_meta_pb.has_visible_ts_ms(); }
487
288
    int64_t visible_ts_ms() const { return _rowset_meta_pb.visible_ts_ms(); }
488
290
    std::chrono::time_point<std::chrono::system_clock> visible_timestamp() const {
489
290
        using namespace std::chrono;
490
290
        if (has_visible_ts_ms()) {
491
288
            return time_point<system_clock>(milliseconds(visible_ts_ms()));
492
288
        }
493
2
        return system_clock::from_time_t(newest_write_timestamp());
494
290
    }
495
177k
    void set_visible_ts_ms(int64_t visible_ts_ms) {
496
177k
        _rowset_meta_pb.set_visible_ts_ms(visible_ts_ms);
497
177k
    }
498
499
    void set_tablet_schema(const TabletSchemaSPtr& tablet_schema);
500
    void set_tablet_schema(const TabletSchemaPB& tablet_schema);
501
502
6.47M
    const TabletSchemaSPtr& tablet_schema() const { return _schema; }
503
504
203k
    void set_txn_expiration(int64_t expiration) { _rowset_meta_pb.set_txn_expiration(expiration); }
505
506
206k
    void set_compaction_level(int64_t compaction_level) {
507
206k
        _rowset_meta_pb.set_compaction_level(compaction_level);
508
206k
    }
509
510
226k
    int64_t compaction_level() { return _rowset_meta_pb.compaction_level(); }
511
512
    // `seg_file_size` MUST be ordered by rowset segment position.
513
    void add_segments_file_size(const std::vector<size_t>& seg_file_size);
514
515
    // Return -1 if segment file size is unknown
516
    int64_t segment_file_size_by_pos(size_t pos) const;
517
518
5.33k
    const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); }
519
520
    // Used for partial update, when publish, partial update may add a new rowset and we should update rowset meta
521
    void merge_rowset_meta(const RowsetMeta& other);
522
523
    InvertedIndexFileInfo inverted_index_file_info_by_pos(size_t pos) const;
524
525
260
    const auto& inverted_index_file_info() const {
526
260
        return _rowset_meta_pb.inverted_index_file_info();
527
260
    }
528
529
    void add_inverted_index_files_info(
530
            const std::vector<const InvertedIndexFileInfo*>& idx_file_info);
531
532
    int64_t get_metadata_size() const override;
533
534
    // Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor
535
    RowsetMeta(const RowsetMeta&) = delete;
536
    RowsetMeta operator=(const RowsetMeta&) = delete;
537
538
    void add_packed_slice_location(const std::string& segment_path,
539
                                   const std::string& packed_file_path, int64_t offset,
540
3
                                   int64_t size, int64_t packed_file_size) {
541
3
        auto* index_map = _rowset_meta_pb.mutable_packed_slice_locations();
542
3
        auto& index_pb = (*index_map)[segment_path];
543
3
        index_pb.set_packed_file_path(packed_file_path);
544
3
        index_pb.set_offset(offset);
545
3
        index_pb.set_size(size);
546
3
        index_pb.set_packed_file_size(packed_file_size);
547
3
    }
548
549
690
    int32_t schema_version() const { return _rowset_meta_pb.schema_version(); }
550
551
0
    std::string debug_string() const { return _rowset_meta_pb.ShortDebugString(); }
552
553
    // Pre-set the encryption algorithm to avoid re-entrant get_tablet calls
554
    // that can cause SingleFlight deadlock during tablet loading.
555
176k
    void set_encryption_algorithm(EncryptionAlgorithmPB algorithm) {
556
176k
        _determine_encryption_once.call(
557
176k
                [algorithm]() -> Result<EncryptionAlgorithmPB> { return algorithm; });
558
176k
    }
559
560
3.12M
    TsoRange commit_tso() const {
561
3.12M
        const auto& commit_tso_pb = _rowset_meta_pb.commit_tso();
562
3.12M
        return {commit_tso_pb.start_tso(), commit_tso_pb.end_tso()};
563
3.12M
    }
564
565
47
    bool has_commit_tso() const { return _rowset_meta_pb.has_commit_tso(); }
566
567
10.6k
    void set_commit_tso(const TsoRange& commit_tso) {
568
10.6k
        auto* commit_tso_pb = _rowset_meta_pb.mutable_commit_tso();
569
10.6k
        commit_tso_pb->set_start_tso(commit_tso.start_tso());
570
10.6k
        commit_tso_pb->set_end_tso(commit_tso.end_tso());
571
10.6k
    }
572
573
467
    void set_commit_tso(int64_t commit_tso) { set_commit_tso({commit_tso, commit_tso}); }
574
575
176k
    void set_cloud_fields_after_visible(int64_t visible_version, int64_t version_update_time_ms) {
576
        // Update rowset meta with correct version and visible_ts
577
        // !!ATTENTION!!: this code should be updated if there are more fields
578
        // in rowset meta which will be modified in meta-service when commit_txn in the future
579
176k
        set_version({visible_version, visible_version});
580
176k
        if (version_update_time_ms > 0) {
581
176k
            set_visible_ts_ms(version_update_time_ms);
582
176k
        }
583
176k
    }
584
585
private:
586
    // Wraps `fs` with `PackedFileSystem` if this rowset has packed slice locations,
587
    // otherwise returns `fs` unchanged.
588
    io::FileSystemSPtr _wrap_packed_fs(io::FileSystemSPtr fs);
589
590
    bool _deserialize_from_pb(std::string_view value);
591
592
    bool _serialize_to_pb(std::string* value);
593
594
    void _init();
595
596
    void _validate_segment_ids() const;
597
598
    friend bool operator==(const RowsetMeta& a, const RowsetMeta& b);
599
600
0
    friend bool operator!=(const RowsetMeta& a, const RowsetMeta& b) { return !(a == b); }
601
602
private:
603
    RowsetMetaPB _rowset_meta_pb;
604
    TabletSchemaSPtr _schema;
605
    Cache::Handle* _handle = nullptr;
606
    RowsetId _rowset_id;
607
    StorageResource _storage_resource;
608
    bool _is_removed_from_rowset_meta = false;
609
    DorisCallOnce<Result<EncryptionAlgorithmPB>> _determine_encryption_once;
610
    std::atomic<int64_t> _stale_at_s {0};
611
};
612
613
class RowsetSegmentMetaView {
614
public:
615
    RowsetSegmentMetaView(const RowsetMeta* meta, size_t pos)
616
3.83M
            : _meta(meta), _ref(meta->segment_ref(pos)) {}
617
618
226k
    size_t pos() const { return _ref.pos; }
619
2.10M
    int64_t id() const { return _ref.id; }
620
1.68M
    RowsetSegmentRef ref() const { return _ref; }
621
622
215k
    int64_t file_size() const { return _meta->segment_file_size_by_pos(pos()); }
623
624
10.7k
    InvertedIndexFileInfo inverted_index_file_info() const {
625
10.7k
        return _meta->inverted_index_file_info_by_pos(pos());
626
10.7k
    }
627
628
    bool has_num_rows() const {
629
        return cast_set<size_t>(_meta->get_num_segment_rows().size()) > pos();
630
    }
631
632
    int64_t num_rows() const {
633
        DORIS_CHECK(has_num_rows());
634
        return _meta->get_num_segment_rows().Get(cast_set<int>(pos()));
635
    }
636
637
    bool has_position_key_bounds() const {
638
        return !_meta->is_segments_key_bounds_aggregated() &&
639
               cast_set<size_t>(_meta->get_segments_key_bounds().size()) > pos();
640
    }
641
642
    const KeyBoundsPB& key_bounds() const {
643
        DORIS_CHECK(has_position_key_bounds());
644
        return _meta->get_segments_key_bounds().Get(cast_set<int>(pos()));
645
    }
646
647
private:
648
    const RowsetMeta* _meta;
649
    RowsetSegmentRef _ref;
650
};
651
652
class RowsetSegmentMetaRange {
653
public:
654
    class Iterator {
655
    public:
656
        using iterator_category = std::forward_iterator_tag;
657
        using value_type = RowsetSegmentMetaView;
658
        using difference_type = std::ptrdiff_t;
659
660
693k
        Iterator(const RowsetMeta* meta, size_t pos) : _meta(meta), _pos(pos) {}
661
662
150k
        RowsetSegmentMetaView operator*() const { return {_meta, _pos}; }
663
664
150k
        Iterator& operator++() {
665
150k
            ++_pos;
666
150k
            return *this;
667
150k
        }
668
669
498k
        bool operator==(const Iterator& other) const {
670
498k
            return _meta == other._meta && _pos == other._pos;
671
498k
        }
672
673
498k
        bool operator!=(const Iterator& other) const { return !(*this == other); }
674
675
    private:
676
        const RowsetMeta* _meta;
677
        size_t _pos;
678
    };
679
680
347k
    explicit RowsetSegmentMetaRange(const RowsetMeta* meta) : _meta(meta) {}
681
682
347k
    Iterator begin() const { return {_meta, 0}; }
683
348k
    Iterator end() const { return {_meta, cast_set<size_t>(_meta->num_segments())}; }
684
685
private:
686
    const RowsetMeta* _meta;
687
};
688
689
3.68M
inline RowsetSegmentMetaView RowsetMeta::segment(size_t pos) const {
690
3.68M
    return {this, pos};
691
3.68M
}
692
693
347k
inline RowsetSegmentMetaRange RowsetMeta::segments() const {
694
347k
    return RowsetSegmentMetaRange(this);
695
347k
}
696
697
} // namespace doris
698
699
#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H