be/src/storage/rowset/rowset_meta.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
19 | | #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
20 | | |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | #include <glog/logging.h> |
23 | | |
24 | | #include <atomic> |
25 | | #include <chrono> |
26 | | #include <cstddef> |
27 | | #include <cstdint> |
28 | | #include <iterator> |
29 | | #include <memory> |
30 | | #include <string> |
31 | | #include <vector> |
32 | | |
33 | | #include "common/cast_set.h" |
34 | | #include "common/check.h" |
35 | | #include "common/config.h" |
36 | | #include "common/status.h" |
37 | | #include "io/fs/encrypted_fs_factory.h" |
38 | | #include "io/fs/file_system.h" |
39 | | #include "runtime/memory/lru_cache_policy.h" |
40 | | #include "storage/metadata_adder.h" |
41 | | #include "storage/olap_common.h" |
42 | | #include "storage/rowset/rowset_fwd.h" |
43 | | #include "storage/rowset/rowset_segment_id.h" |
44 | | #include "storage/storage_policy.h" |
45 | | #include "storage/tablet/tablet_fwd.h" |
46 | | #include "util/once.h" |
47 | | |
48 | | namespace doris { |
49 | | |
50 | | class RowsetSegmentMetaView; |
51 | | class RowsetSegmentMetaRange; |
52 | | |
53 | | class RowsetMeta : public MetadataAdder<RowsetMeta> { |
54 | | public: |
55 | 18.5k | RowsetMeta() = default; |
56 | | ~RowsetMeta(); |
57 | | |
58 | | bool init(std::string_view pb_rowset_meta); |
59 | | |
60 | | bool init(const RowsetMeta* rowset_meta); |
61 | | |
62 | | bool init_from_pb(const RowsetMetaPB& rowset_meta_pb); |
63 | | |
64 | | bool init_from_json(const std::string& json_rowset_meta); |
65 | | |
66 | 6 | bool serialize(std::string* value) { return _serialize_to_pb(value); } |
67 | | |
68 | | bool json_rowset_meta(std::string* json_rowset_meta); |
69 | | |
70 | | // If the rowset is a local rowset, return the global local file system. |
71 | | // Otherwise, return the remote file system corresponding to rowset's resource id. |
72 | | // Note that if the resource id cannot be found for the corresponding remote file system, nullptr will be returned. |
73 | | MOCK_FUNCTION io::FileSystemSPtr fs(); |
74 | | |
75 | | io::FileSystemSPtr physical_fs(); |
76 | | |
77 | | Result<const StorageResource*> remote_storage_resource(); |
78 | | |
79 | | void set_remote_storage_resource(StorageResource resource); |
80 | | |
81 | 5.59k | const std::string& resource_id() const { return _rowset_meta_pb.resource_id(); } |
82 | | |
83 | 22 | void set_resource_id(const std::string& resource_id) { |
84 | 22 | _rowset_meta_pb.set_resource_id(resource_id); |
85 | 22 | } |
86 | | |
87 | 25.9k | bool is_local() const { return !_rowset_meta_pb.has_resource_id(); } |
88 | | |
89 | | bool has_variant_type_in_schema() const; |
90 | | |
91 | 2.54M | RowsetId rowset_id() const { return _rowset_id; } |
92 | | |
93 | 7.88k | void set_rowset_id(const RowsetId& rowset_id) { |
94 | | // rowset id is a required field, just set it to 0 |
95 | 7.88k | _rowset_meta_pb.set_rowset_id(0); |
96 | 7.88k | _rowset_id = rowset_id; |
97 | 7.88k | _rowset_meta_pb.set_rowset_id_v2(rowset_id.to_string()); |
98 | 7.88k | } |
99 | | |
100 | 13.5k | int64_t tablet_id() const { return _rowset_meta_pb.tablet_id(); } |
101 | | |
102 | 6.89k | void set_tablet_id(int64_t tablet_id) { _rowset_meta_pb.set_tablet_id(tablet_id); } |
103 | | |
104 | 44 | int64_t db_id() const { return _rowset_meta_pb.db_id(); } |
105 | | |
106 | 1.31k | void set_db_id(int64_t db_id) { _rowset_meta_pb.set_db_id(db_id); } |
107 | | |
108 | 44 | int64_t table_id() const { return _rowset_meta_pb.table_id(); } |
109 | | |
110 | 1.31k | void set_table_id(int64_t table_id) { _rowset_meta_pb.set_table_id(table_id); } |
111 | | |
112 | 3 | int64_t index_id() const { return _rowset_meta_pb.index_id(); } |
113 | | |
114 | 1.31k | void set_index_id(int64_t index_id) { _rowset_meta_pb.set_index_id(index_id); } |
115 | | |
116 | 24 | TabletUid tablet_uid() const { return _rowset_meta_pb.tablet_uid(); } |
117 | | |
118 | 6.80k | void set_tablet_uid(TabletUid tablet_uid) { |
119 | 6.80k | *(_rowset_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto(); |
120 | 6.80k | } |
121 | | |
122 | 27 | int64_t txn_id() const { return _rowset_meta_pb.txn_id(); } |
123 | | |
124 | 162 | void set_txn_id(int64_t txn_id) { _rowset_meta_pb.set_txn_id(txn_id); } |
125 | | |
126 | 42 | int32_t tablet_schema_hash() const { return _rowset_meta_pb.tablet_schema_hash(); } |
127 | | |
128 | 1.39k | void set_tablet_schema_hash(int32_t tablet_schema_hash) { |
129 | 1.39k | _rowset_meta_pb.set_tablet_schema_hash(tablet_schema_hash); |
130 | 1.39k | } |
131 | | |
132 | 59 | void mark_row_binlog() { _rowset_meta_pb.set_is_row_binlog(true); } |
133 | | |
134 | 1.35k | bool is_row_binlog() const { |
135 | 1.35k | return _rowset_meta_pb.has_is_row_binlog() && _rowset_meta_pb.is_row_binlog(); |
136 | 1.35k | } |
137 | | |
138 | 14.0k | RowsetTypePB rowset_type() const { return _rowset_meta_pb.rowset_type(); } |
139 | | |
140 | 3.47k | void set_rowset_type(RowsetTypePB rowset_type) { _rowset_meta_pb.set_rowset_type(rowset_type); } |
141 | | |
142 | 11.6k | RowsetStatePB rowset_state() const { return _rowset_meta_pb.rowset_state(); } |
143 | | |
144 | 2.72k | void set_rowset_state(RowsetStatePB rowset_state) { |
145 | 2.72k | _rowset_meta_pb.set_rowset_state(rowset_state); |
146 | 2.72k | } |
147 | | |
148 | 2.22M | Version version() const { |
149 | 2.22M | return {_rowset_meta_pb.start_version(), _rowset_meta_pb.end_version()}; |
150 | 2.22M | } |
151 | | |
152 | 7.67k | void set_version(Version version) { |
153 | 7.67k | _rowset_meta_pb.set_start_version(version.first); |
154 | 7.67k | _rowset_meta_pb.set_end_version(version.second); |
155 | 7.67k | } |
156 | | |
157 | 12.8k | bool has_version() const { |
158 | 12.8k | return _rowset_meta_pb.has_start_version() && _rowset_meta_pb.has_end_version(); |
159 | 12.8k | } |
160 | | |
161 | 22.4k | int64_t start_version() const { return _rowset_meta_pb.start_version(); } |
162 | | |
163 | 30.8k | int64_t end_version() const { return _rowset_meta_pb.end_version(); } |
164 | | |
165 | 1.94k | int64_t num_rows() const { return _rowset_meta_pb.num_rows(); } |
166 | | |
167 | 1.81k | void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); } |
168 | | |
169 | 1.37k | void set_num_segment_rows(const std::vector<uint32_t>& num_segment_rows) { |
170 | 1.37k | _rowset_meta_pb.mutable_num_segment_rows()->Assign(num_segment_rows.cbegin(), |
171 | 1.37k | num_segment_rows.cend()); |
172 | 1.37k | } |
173 | | |
174 | 129 | void get_num_segment_rows(std::vector<uint32_t>* num_segment_rows) const { |
175 | 129 | num_segment_rows->assign(_rowset_meta_pb.num_segment_rows().cbegin(), |
176 | 129 | _rowset_meta_pb.num_segment_rows().cend()); |
177 | 129 | } |
178 | | |
179 | 1.64k | auto& get_num_segment_rows() const { return _rowset_meta_pb.num_segment_rows(); } |
180 | | |
181 | 7.43k | int64_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); } |
182 | | |
183 | 9.06k | void set_total_disk_size(int64_t total_disk_size) { |
184 | 9.06k | _rowset_meta_pb.set_total_disk_size(total_disk_size); |
185 | 9.06k | } |
186 | | |
187 | 514 | int64_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); } |
188 | | |
189 | 1.55k | void set_data_disk_size(int64_t data_disk_size) { |
190 | 1.55k | _rowset_meta_pb.set_data_disk_size(data_disk_size); |
191 | 1.55k | } |
192 | | |
193 | 447 | int64_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); } |
194 | | |
195 | 1.50k | void set_index_disk_size(int64_t index_disk_size) { |
196 | 1.50k | _rowset_meta_pb.set_index_disk_size(index_disk_size); |
197 | 1.50k | } |
198 | | |
199 | 0 | void zone_maps(std::vector<::doris::ZoneMap>* zone_maps) { |
200 | 0 | for (const ::doris::ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) { |
201 | 0 | zone_maps->push_back(zone_map); |
202 | 0 | } |
203 | 0 | } |
204 | | |
205 | 0 | void set_zone_maps(const std::vector<::doris::ZoneMap>& zone_maps) { |
206 | 0 | for (const ::doris::ZoneMap& zone_map : zone_maps) { |
207 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
208 | 0 | *new_zone_map = zone_map; |
209 | 0 | } |
210 | 0 | } |
211 | | |
212 | 0 | void add_zone_map(const ::doris::ZoneMap& zone_map) { |
213 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
214 | 0 | *new_zone_map = zone_map; |
215 | 0 | } |
216 | | |
217 | 6.28k | bool has_delete_predicate() const { return _rowset_meta_pb.has_delete_predicate(); } |
218 | | |
219 | 178 | const DeletePredicatePB& delete_predicate() const { return _rowset_meta_pb.delete_predicate(); } |
220 | | |
221 | 0 | DeletePredicatePB* mutable_delete_predicate() { |
222 | 0 | return _rowset_meta_pb.mutable_delete_predicate(); |
223 | 0 | } |
224 | | |
225 | 92 | void set_delete_predicate(DeletePredicatePB delete_predicate) { |
226 | 92 | DeletePredicatePB* new_delete_condition = _rowset_meta_pb.mutable_delete_predicate(); |
227 | 92 | *new_delete_condition = std::move(delete_predicate); |
228 | 92 | } |
229 | | |
230 | 65 | bool empty() const { return _rowset_meta_pb.empty(); } |
231 | | |
232 | 1.49k | void set_empty(bool empty) { _rowset_meta_pb.set_empty(empty); } |
233 | | |
234 | 0 | PUniqueId load_id() const { return _rowset_meta_pb.load_id(); } |
235 | | |
236 | 68 | void set_load_id(PUniqueId load_id) { |
237 | 68 | PUniqueId* new_load_id = _rowset_meta_pb.mutable_load_id(); |
238 | 68 | new_load_id->set_hi(load_id.hi()); |
239 | 68 | new_load_id->set_lo(load_id.lo()); |
240 | 68 | } |
241 | | |
242 | 6 | void set_job_id(const std::string& job_id) { _rowset_meta_pb.set_job_id(job_id); } |
243 | | |
244 | 0 | const std::string& job_id() const { return _rowset_meta_pb.job_id(); } |
245 | | |
246 | 0 | bool delete_flag() const { return _rowset_meta_pb.delete_flag(); } |
247 | | |
248 | 55 | int64_t creation_time() const { return _rowset_meta_pb.creation_time(); } |
249 | | |
250 | 1.46k | void set_creation_time(int64_t creation_time) { |
251 | 1.46k | return _rowset_meta_pb.set_creation_time(creation_time); |
252 | 1.46k | } |
253 | | |
254 | 778 | int64_t stale_at() const { |
255 | 778 | int64_t stale_time = _stale_at_s.load(); |
256 | 778 | return stale_time > 0 ? stale_time : _rowset_meta_pb.creation_time(); |
257 | 778 | } |
258 | | |
259 | 2 | bool has_stale_at() const { return _stale_at_s.load() > 0; } |
260 | | |
261 | 693 | void set_stale_at(int64_t stale_at) { _stale_at_s.store(stale_at); } |
262 | | |
263 | 10 | int64_t partition_id() const { return _rowset_meta_pb.partition_id(); } |
264 | | |
265 | 1.31k | void set_partition_id(int64_t partition_id) { |
266 | 1.31k | return _rowset_meta_pb.set_partition_id(partition_id); |
267 | 1.31k | } |
268 | | |
269 | 66.3k | int64_t num_segments() const { |
270 | 66.3k | DCHECK(_rowset_meta_pb.segment_ids_size() == 0 || |
271 | 66.3k | _rowset_meta_pb.segment_ids_size() == _rowset_meta_pb.num_segments()); |
272 | 66.3k | return _rowset_meta_pb.num_segments(); |
273 | 66.3k | } |
274 | | |
275 | 5.89k | void set_num_segments(int64_t num_segments) { _rowset_meta_pb.set_num_segments(num_segments); } |
276 | | |
277 | 18.7k | bool has_segment_ids() const { return _rowset_meta_pb.segment_ids_size() > 0; } |
278 | | |
279 | 3 | const auto& segment_ids() const { return _rowset_meta_pb.segment_ids(); } |
280 | | |
281 | | void set_segment_ids(const std::vector<int64_t>& segment_ids); |
282 | | |
283 | 8.20k | int64_t segment_id(size_t pos) const { |
284 | 8.20k | DORIS_CHECK_LT(pos, cast_set<size_t>(num_segments())); |
285 | 8.20k | return has_segment_ids() ? _rowset_meta_pb.segment_ids(cast_set<int>(pos)) |
286 | 8.20k | : cast_set<int64_t>(pos); |
287 | 8.20k | } |
288 | | |
289 | 8.19k | RowsetSegmentRef segment_ref(size_t pos) const { return {pos, segment_id(pos)}; } |
290 | | |
291 | | RowsetSegmentMetaView segment(size_t pos) const; |
292 | | |
293 | | RowsetSegmentMetaRange segments() const; |
294 | | |
295 | | size_t position_of(int64_t seg_id) const; |
296 | | |
297 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
298 | | void to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema = false) const; |
299 | | |
300 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
301 | | RowsetMetaPB get_rowset_pb(bool skip_schema = false) const; |
302 | | |
303 | 0 | inline DeletePredicatePB* mutable_delete_pred_pb() { |
304 | 0 | return _rowset_meta_pb.mutable_delete_predicate(); |
305 | 0 | } |
306 | | |
307 | 220 | bool is_singleton_delta() const { |
308 | 220 | return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version(); |
309 | 220 | } |
310 | | |
311 | | // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta. |
312 | | // But, this check behavior may cost a lot of time when it is frequent. |
313 | | // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true, |
314 | | // And next time when we want to check if this rowset is in rowset mata manager's meta, we can |
315 | | // check is_remove_from_rowset_meta() first. |
316 | 0 | void set_remove_from_rowset_meta() { _is_removed_from_rowset_meta = true; } |
317 | | |
318 | 0 | bool is_remove_from_rowset_meta() const { return _is_removed_from_rowset_meta; } |
319 | | |
320 | 1.92k | SegmentsOverlapPB segments_overlap() const { return _rowset_meta_pb.segments_overlap_pb(); } |
321 | | |
322 | 6.50k | void set_segments_overlap(SegmentsOverlapPB segments_overlap) { |
323 | 6.50k | _rowset_meta_pb.set_segments_overlap_pb(segments_overlap); |
324 | 6.50k | } |
325 | | |
326 | 7.80k | static bool comparator(const RowsetMetaSharedPtr& left, const RowsetMetaSharedPtr& right) { |
327 | 7.80k | return left->end_version() < right->end_version(); |
328 | 7.80k | } |
329 | | |
330 | | // return true if segments in this rowset has overlapping data. |
331 | | // this is not same as `segments_overlap()` method. |
332 | | // `segments_overlap()` only return the value of "segments_overlap" field in rowset meta, |
333 | | // but "segments_overlap" may be UNKNOWN. |
334 | | // |
335 | | // Returns true if all of the following conditions are met |
336 | | // 1. the rowset contains more than one segment |
337 | | // 2. the rowset's start version == end version (non-singleton rowset was generated by compaction process |
338 | | // which always produces non-overlapped segments) |
339 | | // 3. segments_overlap() flag is not NONOVERLAPPING (OVERLAP_UNKNOWN and OVERLAPPING are OK) |
340 | | // 4. relaxing is_singleton_delta() for is_row_binlog as Row-binlog LMax quick merge does produce |
341 | | // non-singleton overlapping segments, but it writes segments_overlap = OVERLAPPING explicitly |
342 | 20.2k | bool is_segments_overlapping() const { |
343 | 20.2k | return num_segments() > 1 && segments_overlap() != NONOVERLAPPING && |
344 | 20.2k | (is_singleton_delta() || (is_row_binlog() && segments_overlap() == OVERLAPPING)); |
345 | 20.2k | } |
346 | | |
347 | 0 | bool produced_by_compaction() const { |
348 | 0 | return has_version() && |
349 | 0 | (start_version() < end_version() || |
350 | 0 | (start_version() == end_version() && segments_overlap() == NONOVERLAPPING)); |
351 | 0 | } |
352 | | |
353 | | // get the compaction score of this rowset. |
354 | | // if segments are overlapping, the score equals to the number of segments, |
355 | | // otherwise, score is 1. |
356 | 18.2k | uint32_t get_compaction_score() const { |
357 | 18.2k | uint32_t score = 0; |
358 | 18.2k | if (!is_segments_overlapping()) { |
359 | 18.0k | score = 1; |
360 | 18.0k | } else { |
361 | 132 | auto num_seg = num_segments(); |
362 | 132 | DCHECK_GT(num_seg, 0); |
363 | 132 | score = cast_set<uint32_t>(num_seg); |
364 | 132 | CHECK(score > 0); |
365 | 132 | } |
366 | 18.2k | return score; |
367 | 18.2k | } |
368 | | |
369 | 55 | uint32_t get_merge_way_num() const { |
370 | 55 | uint32_t way_num = 0; |
371 | 55 | if (!is_segments_overlapping()) { |
372 | 55 | if (num_segments() == 0) { |
373 | 0 | way_num = 0; |
374 | 55 | } else { |
375 | 55 | way_num = 1; |
376 | 55 | } |
377 | 55 | } else { |
378 | 0 | auto num_seg = num_segments(); |
379 | 0 | DCHECK_GT(num_seg, 0); |
380 | |
|
381 | 0 | way_num = cast_set<uint32_t>(num_seg); |
382 | 0 | CHECK(way_num > 0); |
383 | 0 | } |
384 | 55 | return way_num; |
385 | 55 | } |
386 | | |
387 | 582 | void get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) const { |
388 | 601 | for (const KeyBoundsPB& key_range : _rowset_meta_pb.segments_key_bounds()) { |
389 | 601 | segments_key_bounds->push_back(key_range); |
390 | 601 | } |
391 | 582 | } |
392 | | |
393 | 8 | auto& get_segments_key_bounds() const { return _rowset_meta_pb.segments_key_bounds(); } |
394 | | |
395 | 2.24k | bool is_segments_key_bounds_truncated() const { |
396 | 2.24k | return _rowset_meta_pb.has_segments_key_bounds_truncated() && |
397 | 2.24k | _rowset_meta_pb.segments_key_bounds_truncated(); |
398 | 2.24k | } |
399 | | |
400 | 2.01k | void set_segments_key_bounds_truncated(bool truncated) { |
401 | 2.01k | _rowset_meta_pb.set_segments_key_bounds_truncated(truncated); |
402 | 2.01k | } |
403 | | |
404 | | // When true, `segments_key_bounds` holds a single aggregated |
405 | | // [rowset_min, rowset_max] entry instead of per-segment bounds. |
406 | 461 | bool is_segments_key_bounds_aggregated() const { |
407 | 461 | return _rowset_meta_pb.has_segments_key_bounds_aggregated() && |
408 | 461 | _rowset_meta_pb.segments_key_bounds_aggregated(); |
409 | 461 | } |
410 | | |
411 | 1.37k | void set_segments_key_bounds_aggregated(bool aggregated) { |
412 | 1.37k | _rowset_meta_pb.set_segments_key_bounds_aggregated(aggregated); |
413 | 1.37k | } |
414 | | |
415 | 263 | bool get_first_segment_key_bound(KeyBoundsPB* key_bounds) { |
416 | | // for compatibility, old version has not segment key bounds |
417 | 263 | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
418 | 0 | return false; |
419 | 0 | } |
420 | 263 | *key_bounds = *_rowset_meta_pb.segments_key_bounds().begin(); |
421 | 263 | return true; |
422 | 263 | } |
423 | | |
424 | 175 | bool get_last_segment_key_bound(KeyBoundsPB* key_bounds) { |
425 | 175 | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
426 | 0 | return false; |
427 | 0 | } |
428 | 175 | *key_bounds = *_rowset_meta_pb.segments_key_bounds().rbegin(); |
429 | 175 | return true; |
430 | 175 | } |
431 | | |
432 | | // If `aggregate_into_single` is true, collapse per-segment bounds into a single |
433 | | // [rowset_min, rowset_max] entry and mark this rowset as aggregated. |
434 | | void set_segments_key_bounds(const std::vector<KeyBoundsPB>& segments_key_bounds, |
435 | | bool aggregate_into_single = false); |
436 | | |
437 | 0 | void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) { |
438 | 0 | *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds); |
439 | 0 | set_segments_overlap(OVERLAPPING); |
440 | 0 | } |
441 | | |
442 | 2.16k | void set_newest_write_timestamp(int64_t timestamp) { |
443 | 2.16k | _rowset_meta_pb.set_newest_write_timestamp(timestamp); |
444 | 2.16k | } |
445 | | |
446 | 1.40k | int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } |
447 | | |
448 | | // for cloud only |
449 | 269 | bool has_visible_ts_ms() const { return _rowset_meta_pb.has_visible_ts_ms(); } |
450 | 268 | int64_t visible_ts_ms() const { return _rowset_meta_pb.visible_ts_ms(); } |
451 | 269 | std::chrono::time_point<std::chrono::system_clock> visible_timestamp() const { |
452 | 269 | using namespace std::chrono; |
453 | 269 | if (has_visible_ts_ms()) { |
454 | 268 | return time_point<system_clock>(milliseconds(visible_ts_ms())); |
455 | 268 | } |
456 | 1 | return system_clock::from_time_t(newest_write_timestamp()); |
457 | 269 | } |
458 | 771 | void set_visible_ts_ms(int64_t visible_ts_ms) { |
459 | 771 | _rowset_meta_pb.set_visible_ts_ms(visible_ts_ms); |
460 | 771 | } |
461 | | |
462 | | void set_tablet_schema(const TabletSchemaSPtr& tablet_schema); |
463 | | void set_tablet_schema(const TabletSchemaPB& tablet_schema); |
464 | | |
465 | 31.4k | const TabletSchemaSPtr& tablet_schema() const { return _schema; } |
466 | | |
467 | 6 | void set_txn_expiration(int64_t expiration) { _rowset_meta_pb.set_txn_expiration(expiration); } |
468 | | |
469 | 1.31k | void set_compaction_level(int64_t compaction_level) { |
470 | 1.31k | _rowset_meta_pb.set_compaction_level(compaction_level); |
471 | 1.31k | } |
472 | | |
473 | 878 | int64_t compaction_level() { return _rowset_meta_pb.compaction_level(); } |
474 | | |
475 | | // `seg_file_size` MUST be ordered by rowset segment position. |
476 | | void add_segments_file_size(const std::vector<size_t>& seg_file_size); |
477 | | |
478 | | // Return -1 if segment file size is unknown |
479 | | int64_t segment_file_size_by_pos(size_t pos) const; |
480 | | |
481 | 0 | const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); } |
482 | | |
483 | | // Used for partial update, when publish, partial update may add a new rowset and we should update rowset meta |
484 | | void merge_rowset_meta(const RowsetMeta& other); |
485 | | |
486 | | InvertedIndexFileInfo inverted_index_file_info_by_pos(size_t pos) const; |
487 | | |
488 | 66 | const auto& inverted_index_file_info() const { |
489 | 66 | return _rowset_meta_pb.inverted_index_file_info(); |
490 | 66 | } |
491 | | |
492 | | void add_inverted_index_files_info( |
493 | | const std::vector<const InvertedIndexFileInfo*>& idx_file_info); |
494 | | |
495 | | int64_t get_metadata_size() const override; |
496 | | |
497 | | // Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor |
498 | | RowsetMeta(const RowsetMeta&) = delete; |
499 | | RowsetMeta operator=(const RowsetMeta&) = delete; |
500 | | |
501 | | void add_packed_slice_location(const std::string& segment_path, |
502 | | const std::string& packed_file_path, int64_t offset, |
503 | 0 | int64_t size, int64_t packed_file_size) { |
504 | 0 | auto* index_map = _rowset_meta_pb.mutable_packed_slice_locations(); |
505 | 0 | auto& index_pb = (*index_map)[segment_path]; |
506 | 0 | index_pb.set_packed_file_path(packed_file_path); |
507 | 0 | index_pb.set_offset(offset); |
508 | 0 | index_pb.set_size(size); |
509 | 0 | index_pb.set_packed_file_size(packed_file_size); |
510 | 0 | } |
511 | | |
512 | 148 | int32_t schema_version() const { return _rowset_meta_pb.schema_version(); } |
513 | | |
514 | 0 | std::string debug_string() const { return _rowset_meta_pb.ShortDebugString(); } |
515 | | |
516 | | // Pre-set the encryption algorithm to avoid re-entrant get_tablet calls |
517 | | // that can cause SingleFlight deadlock during tablet loading. |
518 | 0 | void set_encryption_algorithm(EncryptionAlgorithmPB algorithm) { |
519 | 0 | _determine_encryption_once.call( |
520 | 0 | [algorithm]() -> Result<EncryptionAlgorithmPB> { return algorithm; }); |
521 | 0 | } |
522 | | |
523 | 1.68k | TsoRange commit_tso() const { |
524 | 1.68k | const auto& commit_tso_pb = _rowset_meta_pb.commit_tso(); |
525 | 1.68k | return {commit_tso_pb.start_tso(), commit_tso_pb.end_tso()}; |
526 | 1.68k | } |
527 | | |
528 | 44 | bool has_commit_tso() const { return _rowset_meta_pb.has_commit_tso(); } |
529 | | |
530 | 76 | void set_commit_tso(const TsoRange& commit_tso) { |
531 | 76 | auto* commit_tso_pb = _rowset_meta_pb.mutable_commit_tso(); |
532 | 76 | commit_tso_pb->set_start_tso(commit_tso.start_tso()); |
533 | 76 | commit_tso_pb->set_end_tso(commit_tso.end_tso()); |
534 | 76 | } |
535 | | |
536 | 19 | void set_commit_tso(int64_t commit_tso) { set_commit_tso({commit_tso, commit_tso}); } |
537 | | |
538 | 0 | void set_cloud_fields_after_visible(int64_t visible_version, int64_t version_update_time_ms) { |
539 | | // Update rowset meta with correct version and visible_ts |
540 | | // !!ATTENTION!!: this code should be updated if there are more fields |
541 | | // in rowset meta which will be modified in meta-service when commit_txn in the future |
542 | 0 | set_version({visible_version, visible_version}); |
543 | 0 | if (version_update_time_ms > 0) { |
544 | 0 | set_visible_ts_ms(version_update_time_ms); |
545 | 0 | } |
546 | 0 | } |
547 | | |
548 | | private: |
549 | | bool _deserialize_from_pb(std::string_view value); |
550 | | |
551 | | bool _serialize_to_pb(std::string* value); |
552 | | |
553 | | void _init(); |
554 | | |
555 | | void _validate_segment_ids() const; |
556 | | |
557 | | friend bool operator==(const RowsetMeta& a, const RowsetMeta& b); |
558 | | |
559 | 0 | friend bool operator!=(const RowsetMeta& a, const RowsetMeta& b) { return !(a == b); } |
560 | | |
561 | | private: |
562 | | RowsetMetaPB _rowset_meta_pb; |
563 | | TabletSchemaSPtr _schema; |
564 | | Cache::Handle* _handle = nullptr; |
565 | | RowsetId _rowset_id; |
566 | | StorageResource _storage_resource; |
567 | | bool _is_removed_from_rowset_meta = false; |
568 | | DorisCallOnce<Result<EncryptionAlgorithmPB>> _determine_encryption_once; |
569 | | std::atomic<int64_t> _stale_at_s {0}; |
570 | | }; |
571 | | |
572 | | class RowsetSegmentMetaView { |
573 | | public: |
574 | | RowsetSegmentMetaView(const RowsetMeta* meta, size_t pos) |
575 | 8.13k | : _meta(meta), _ref(meta->segment_ref(pos)) {} |
576 | | |
577 | 1.67k | size_t pos() const { return _ref.pos; } |
578 | 2.86k | int64_t id() const { return _ref.id; } |
579 | 5.48k | RowsetSegmentRef ref() const { return _ref; } |
580 | | |
581 | 1 | int64_t file_size() const { return _meta->segment_file_size_by_pos(pos()); } |
582 | | |
583 | 1.66k | InvertedIndexFileInfo inverted_index_file_info() const { |
584 | 1.66k | return _meta->inverted_index_file_info_by_pos(pos()); |
585 | 1.66k | } |
586 | | |
587 | 2 | bool has_num_rows() const { |
588 | 2 | return cast_set<size_t>(_meta->get_num_segment_rows().size()) > pos(); |
589 | 2 | } |
590 | | |
591 | 1 | int64_t num_rows() const { |
592 | 1 | DORIS_CHECK(has_num_rows()); |
593 | 1 | return _meta->get_num_segment_rows().Get(cast_set<int>(pos())); |
594 | 1 | } |
595 | | |
596 | 3 | bool has_position_key_bounds() const { |
597 | 3 | return !_meta->is_segments_key_bounds_aggregated() && |
598 | 3 | cast_set<size_t>(_meta->get_segments_key_bounds().size()) > pos(); |
599 | 3 | } |
600 | | |
601 | 2 | const KeyBoundsPB& key_bounds() const { |
602 | 2 | DORIS_CHECK(has_position_key_bounds()); |
603 | 2 | return _meta->get_segments_key_bounds().Get(cast_set<int>(pos())); |
604 | 2 | } |
605 | | |
606 | | private: |
607 | | const RowsetMeta* _meta; |
608 | | RowsetSegmentRef _ref; |
609 | | }; |
610 | | |
611 | | class RowsetSegmentMetaRange { |
612 | | public: |
613 | | class Iterator { |
614 | | public: |
615 | | using iterator_category = std::forward_iterator_tag; |
616 | | using value_type = RowsetSegmentMetaView; |
617 | | using difference_type = std::ptrdiff_t; |
618 | | |
619 | 22.2k | Iterator(const RowsetMeta* meta, size_t pos) : _meta(meta), _pos(pos) {} |
620 | | |
621 | 351 | RowsetSegmentMetaView operator*() const { return {_meta, _pos}; } |
622 | | |
623 | 348 | Iterator& operator++() { |
624 | 348 | ++_pos; |
625 | 348 | return *this; |
626 | 348 | } |
627 | | |
628 | 11.4k | bool operator==(const Iterator& other) const { |
629 | 11.4k | return _meta == other._meta && _pos == other._pos; |
630 | 11.4k | } |
631 | | |
632 | 11.4k | bool operator!=(const Iterator& other) const { return !(*this == other); } |
633 | | |
634 | | private: |
635 | | const RowsetMeta* _meta; |
636 | | size_t _pos; |
637 | | }; |
638 | | |
639 | 11.1k | explicit RowsetSegmentMetaRange(const RowsetMeta* meta) : _meta(meta) {} |
640 | | |
641 | 11.1k | Iterator begin() const { return {_meta, 0}; } |
642 | 11.1k | Iterator end() const { return {_meta, cast_set<size_t>(_meta->num_segments())}; } |
643 | | |
644 | | private: |
645 | | const RowsetMeta* _meta; |
646 | | }; |
647 | | |
648 | 7.78k | inline RowsetSegmentMetaView RowsetMeta::segment(size_t pos) const { |
649 | 7.78k | return {this, pos}; |
650 | 7.78k | } |
651 | | |
652 | 11.1k | inline RowsetSegmentMetaRange RowsetMeta::segments() const { |
653 | 11.1k | return RowsetSegmentMetaRange(this); |
654 | 11.1k | } |
655 | | |
656 | | } // namespace doris |
657 | | |
658 | | #endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |