be/src/storage/rowset/rowset_meta.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
19 | | #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
20 | | |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | #include <glog/logging.h> |
23 | | |
24 | | #include <atomic> |
25 | | #include <chrono> |
26 | | #include <cstddef> |
27 | | #include <cstdint> |
28 | | #include <iterator> |
29 | | #include <memory> |
30 | | #include <string> |
31 | | #include <vector> |
32 | | |
33 | | #include "common/cast_set.h" |
34 | | #include "common/check.h" |
35 | | #include "common/config.h" |
36 | | #include "common/status.h" |
37 | | #include "io/fs/encrypted_fs_factory.h" |
38 | | #include "io/fs/file_system.h" |
39 | | #include "runtime/memory/lru_cache_policy.h" |
40 | | #include "storage/metadata_adder.h" |
41 | | #include "storage/olap_common.h" |
42 | | #include "storage/rowset/rowset_fwd.h" |
43 | | #include "storage/rowset/rowset_segment_id.h" |
44 | | #include "storage/storage_policy.h" |
45 | | #include "storage/tablet/tablet_fwd.h" |
46 | | #include "util/once.h" |
47 | | |
48 | | namespace doris { |
49 | | |
50 | | class RowsetSegmentMetaView; |
51 | | class RowsetSegmentMetaRange; |
52 | | |
53 | | class RowsetMeta : public MetadataAdder<RowsetMeta> { |
54 | | public: |
55 | 1.06M | RowsetMeta() = default; |
56 | | ~RowsetMeta(); |
57 | | |
58 | | bool init(std::string_view pb_rowset_meta); |
59 | | |
60 | | bool init(const RowsetMeta* rowset_meta); |
61 | | |
62 | | bool init_from_pb(const RowsetMetaPB& rowset_meta_pb); |
63 | | |
64 | | bool init_from_json(const std::string& json_rowset_meta); |
65 | | |
66 | 6 | bool serialize(std::string* value) { return _serialize_to_pb(value); } |
67 | | |
68 | | bool json_rowset_meta(std::string* json_rowset_meta); |
69 | | |
70 | | // If the rowset is a local rowset, return the global local file system. |
71 | | // Otherwise, return the remote file system corresponding to rowset's resource id. |
72 | | // Note that if the resource id cannot be found for the corresponding remote file system, nullptr will be returned. |
73 | | MOCK_FUNCTION io::FileSystemSPtr fs(); |
74 | | |
75 | | io::FileSystemSPtr physical_fs(); |
76 | | |
77 | | Result<const StorageResource*> remote_storage_resource(); |
78 | | |
79 | | void set_remote_storage_resource(StorageResource resource); |
80 | | |
81 | 1.79M | const std::string& resource_id() const { return _rowset_meta_pb.resource_id(); } |
82 | | |
83 | 1.79k | void set_resource_id(const std::string& resource_id) { |
84 | 1.79k | _rowset_meta_pb.set_resource_id(resource_id); |
85 | 1.79k | } |
86 | | |
87 | 47.2M | bool is_local() const { return !_rowset_meta_pb.has_resource_id(); } |
88 | | |
89 | | bool has_variant_type_in_schema() const; |
90 | | |
91 | 24.8M | RowsetId rowset_id() const { return _rowset_id; } |
92 | | |
93 | 205k | void set_rowset_id(const RowsetId& rowset_id) { |
94 | | // rowset id is a required field, just set it to 0 |
95 | 205k | _rowset_meta_pb.set_rowset_id(0); |
96 | 205k | _rowset_id = rowset_id; |
97 | 205k | _rowset_meta_pb.set_rowset_id_v2(rowset_id.to_string()); |
98 | 205k | } |
99 | | |
100 | 8.86M | int64_t tablet_id() const { return _rowset_meta_pb.tablet_id(); } |
101 | | |
102 | 205k | void set_tablet_id(int64_t tablet_id) { _rowset_meta_pb.set_tablet_id(tablet_id); } |
103 | | |
104 | 3.10k | int64_t db_id() const { return _rowset_meta_pb.db_id(); } |
105 | | |
106 | 198k | void set_db_id(int64_t db_id) { _rowset_meta_pb.set_db_id(db_id); } |
107 | | |
108 | 3.04k | int64_t table_id() const { return _rowset_meta_pb.table_id(); } |
109 | | |
110 | 198k | void set_table_id(int64_t table_id) { _rowset_meta_pb.set_table_id(table_id); } |
111 | | |
112 | | int64_t index_id() const { return _rowset_meta_pb.index_id(); } |
113 | | |
114 | 200k | void set_index_id(int64_t index_id) { _rowset_meta_pb.set_index_id(index_id); } |
115 | | |
116 | | bool has_inverted_index_storage_format() const { |
117 | | return _rowset_meta_pb.has_inverted_index_storage_format(); |
118 | | } |
119 | | |
120 | | InvertedIndexStorageFormatPB inverted_index_storage_format() const { |
121 | | return _rowset_meta_pb.inverted_index_storage_format(); |
122 | | } |
123 | | |
124 | | void set_inverted_index_storage_format(InvertedIndexStorageFormatPB format); |
125 | | |
126 | 2.06k | TabletUid tablet_uid() const { return _rowset_meta_pb.tablet_uid(); } |
127 | | |
128 | 8.93k | void set_tablet_uid(TabletUid tablet_uid) { |
129 | 8.93k | *(_rowset_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto(); |
130 | 8.93k | } |
131 | | |
132 | 665k | int64_t txn_id() const { return _rowset_meta_pb.txn_id(); } |
133 | | |
134 | 197k | void set_txn_id(int64_t txn_id) { _rowset_meta_pb.set_txn_id(txn_id); } |
135 | | |
136 | 1.98k | int32_t tablet_schema_hash() const { return _rowset_meta_pb.tablet_schema_hash(); } |
137 | | |
138 | 199k | void set_tablet_schema_hash(int32_t tablet_schema_hash) { |
139 | 199k | _rowset_meta_pb.set_tablet_schema_hash(tablet_schema_hash); |
140 | 199k | } |
141 | | |
142 | 139 | void mark_row_binlog() { _rowset_meta_pb.set_is_row_binlog(true); } |
143 | | |
144 | 143k | bool is_row_binlog() const { |
145 | 143k | return _rowset_meta_pb.has_is_row_binlog() && _rowset_meta_pb.is_row_binlog(); |
146 | 143k | } |
147 | | |
148 | 1.93M | RowsetTypePB rowset_type() const { return _rowset_meta_pb.rowset_type(); } |
149 | | |
150 | 201k | void set_rowset_type(RowsetTypePB rowset_type) { _rowset_meta_pb.set_rowset_type(rowset_type); } |
151 | | |
152 | 548k | RowsetStatePB rowset_state() const { return _rowset_meta_pb.rowset_state(); } |
153 | | |
154 | 402k | void set_rowset_state(RowsetStatePB rowset_state) { |
155 | 402k | _rowset_meta_pb.set_rowset_state(rowset_state); |
156 | 402k | } |
157 | | |
158 | 48.4M | Version version() const { |
159 | 48.4M | return {_rowset_meta_pb.start_version(), _rowset_meta_pb.end_version()}; |
160 | 48.4M | } |
161 | | |
162 | 251k | void set_version(Version version) { |
163 | 251k | _rowset_meta_pb.set_start_version(version.first); |
164 | 251k | _rowset_meta_pb.set_end_version(version.second); |
165 | 251k | } |
166 | | |
167 | 986k | bool has_version() const { |
168 | 986k | return _rowset_meta_pb.has_start_version() && _rowset_meta_pb.has_end_version(); |
169 | 986k | } |
170 | | |
171 | 4.89M | int64_t start_version() const { return _rowset_meta_pb.start_version(); } |
172 | | |
173 | 7.82M | int64_t end_version() const { return _rowset_meta_pb.end_version(); } |
174 | | |
175 | 8.86M | int64_t num_rows() const { return _rowset_meta_pb.num_rows(); } |
176 | | |
177 | 225k | void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); } |
178 | | |
179 | 219k | void set_num_segment_rows(const std::vector<uint32_t>& num_segment_rows) { |
180 | 219k | _rowset_meta_pb.mutable_num_segment_rows()->Assign(num_segment_rows.cbegin(), |
181 | 219k | num_segment_rows.cend()); |
182 | 219k | } |
183 | | |
184 | 131 | void get_num_segment_rows(std::vector<uint32_t>* num_segment_rows) const { |
185 | 131 | num_segment_rows->assign(_rowset_meta_pb.num_segment_rows().cbegin(), |
186 | 131 | _rowset_meta_pb.num_segment_rows().cend()); |
187 | 131 | } |
188 | | |
189 | 147k | auto& get_num_segment_rows() const { return _rowset_meta_pb.num_segment_rows(); } |
190 | | |
191 | 14 | void set_segment_group_sizes(const std::vector<int32_t>& segment_group_sizes) { |
192 | 14 | DORIS_CHECK_GT(segment_group_sizes.size(), 1); |
193 | 14 | int64_t segment_count = 0; |
194 | 45 | for (const auto group_size : segment_group_sizes) { |
195 | 45 | DORIS_CHECK_GT(group_size, 0); |
196 | 45 | segment_count += group_size; |
197 | 45 | } |
198 | 14 | DORIS_CHECK_EQ(segment_count, num_segments()); |
199 | 14 | _rowset_meta_pb.mutable_segment_group_sizes()->Assign(segment_group_sizes.cbegin(), |
200 | 14 | segment_group_sizes.cend()); |
201 | 14 | } |
202 | | |
203 | | void clear_segment_group_sizes() { _rowset_meta_pb.clear_segment_group_sizes(); } |
204 | | |
205 | 34 | const auto& segment_group_sizes() const { return _rowset_meta_pb.segment_group_sizes(); } |
206 | | |
207 | 10.3M | int64_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); } |
208 | | |
209 | 236k | void set_total_disk_size(int64_t total_disk_size) { |
210 | 236k | _rowset_meta_pb.set_total_disk_size(total_disk_size); |
211 | 236k | } |
212 | | |
213 | 8.81M | int64_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); } |
214 | | |
215 | 225k | void set_data_disk_size(int64_t data_disk_size) { |
216 | 225k | _rowset_meta_pb.set_data_disk_size(data_disk_size); |
217 | 225k | } |
218 | | |
219 | 2.47M | int64_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); } |
220 | | |
221 | 225k | void set_index_disk_size(int64_t index_disk_size) { |
222 | 225k | _rowset_meta_pb.set_index_disk_size(index_disk_size); |
223 | 225k | } |
224 | | |
225 | 0 | void zone_maps(std::vector<::doris::ZoneMap>* zone_maps) { |
226 | 0 | for (const ::doris::ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) { |
227 | 0 | zone_maps->push_back(zone_map); |
228 | 0 | } |
229 | 0 | } |
230 | | |
231 | 0 | void set_zone_maps(const std::vector<::doris::ZoneMap>& zone_maps) { |
232 | 0 | for (const ::doris::ZoneMap& zone_map : zone_maps) { |
233 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
234 | 0 | *new_zone_map = zone_map; |
235 | 0 | } |
236 | 0 | } |
237 | | |
238 | 0 | void add_zone_map(const ::doris::ZoneMap& zone_map) { |
239 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
240 | 0 | *new_zone_map = zone_map; |
241 | 0 | } |
242 | | |
243 | 5.02M | bool has_delete_predicate() const { return _rowset_meta_pb.has_delete_predicate(); } |
244 | | |
245 | 8.23k | const DeletePredicatePB& delete_predicate() const { return _rowset_meta_pb.delete_predicate(); } |
246 | | |
247 | 0 | DeletePredicatePB* mutable_delete_predicate() { |
248 | 0 | return _rowset_meta_pb.mutable_delete_predicate(); |
249 | 0 | } |
250 | | |
251 | 3.24k | void set_delete_predicate(DeletePredicatePB delete_predicate) { |
252 | 3.24k | DeletePredicatePB* new_delete_condition = _rowset_meta_pb.mutable_delete_predicate(); |
253 | 3.24k | *new_delete_condition = std::move(delete_predicate); |
254 | 3.24k | } |
255 | | |
256 | 8.35k | bool empty() const { return _rowset_meta_pb.empty(); } |
257 | | |
258 | 221k | void set_empty(bool empty) { _rowset_meta_pb.set_empty(empty); } |
259 | | |
260 | 174 | PUniqueId load_id() const { return _rowset_meta_pb.load_id(); } |
261 | | |
262 | 180k | void set_load_id(PUniqueId load_id) { |
263 | 180k | PUniqueId* new_load_id = _rowset_meta_pb.mutable_load_id(); |
264 | 180k | new_load_id->set_hi(load_id.hi()); |
265 | 180k | new_load_id->set_lo(load_id.lo()); |
266 | 180k | } |
267 | | |
268 | 198k | void set_job_id(const std::string& job_id) { _rowset_meta_pb.set_job_id(job_id); } |
269 | | |
270 | 0 | const std::string& job_id() const { return _rowset_meta_pb.job_id(); } |
271 | | |
272 | 0 | bool delete_flag() const { return _rowset_meta_pb.delete_flag(); } |
273 | | |
274 | 130k | int64_t creation_time() const { return _rowset_meta_pb.creation_time(); } |
275 | | |
276 | 221k | void set_creation_time(int64_t creation_time) { |
277 | 221k | return _rowset_meta_pb.set_creation_time(creation_time); |
278 | 221k | } |
279 | | |
280 | 416k | int64_t stale_at() const { |
281 | 416k | int64_t stale_time = _stale_at_s.load(); |
282 | 416k | return stale_time > 0 ? stale_time : _rowset_meta_pb.creation_time(); |
283 | 416k | } |
284 | | |
285 | 834 | bool has_stale_at() const { return _stale_at_s.load() > 0; } |
286 | | |
287 | 73.2k | void set_stale_at(int64_t stale_at) { _stale_at_s.store(stale_at); } |
288 | | |
289 | 1.66k | int64_t partition_id() const { return _rowset_meta_pb.partition_id(); } |
290 | | |
291 | 199k | void set_partition_id(int64_t partition_id) { |
292 | 199k | return _rowset_meta_pb.set_partition_id(partition_id); |
293 | 199k | } |
294 | | |
295 | 31.0M | int64_t num_segments() const { |
296 | 31.0M | DCHECK(_rowset_meta_pb.segment_ids_size() == 0 || |
297 | 31.0M | _rowset_meta_pb.segment_ids_size() == _rowset_meta_pb.num_segments()); |
298 | 31.0M | return _rowset_meta_pb.num_segments(); |
299 | 31.0M | } |
300 | | |
301 | 258k | void set_num_segments(int64_t num_segments) { _rowset_meta_pb.set_num_segments(num_segments); } |
302 | | |
303 | 4.69M | bool has_segment_ids() const { return _rowset_meta_pb.segment_ids_size() > 0; } |
304 | | |
305 | 274k | const auto& segment_ids() const { return _rowset_meta_pb.segment_ids(); } |
306 | | |
307 | | void set_segment_ids(const std::vector<int64_t>& segment_ids); |
308 | | |
309 | 3.75M | int64_t segment_id(size_t pos) const { |
310 | 3.75M | DORIS_CHECK_LT(pos, cast_set<size_t>(num_segments())); |
311 | 3.75M | return has_segment_ids() ? _rowset_meta_pb.segment_ids(cast_set<int>(pos)) |
312 | 3.75M | : cast_set<int64_t>(pos); |
313 | 3.75M | } |
314 | | |
315 | 3.76M | RowsetSegmentRef segment_ref(size_t pos) const { return {pos, segment_id(pos)}; } |
316 | | |
317 | | RowsetSegmentMetaView segment(size_t pos) const; |
318 | | |
319 | | RowsetSegmentMetaRange segments() const; |
320 | | |
321 | | size_t position_of(int64_t seg_id) const; |
322 | | |
323 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
324 | | void to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema = false) const; |
325 | | |
326 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
327 | | RowsetMetaPB get_rowset_pb(bool skip_schema = false) const; |
328 | | |
329 | 0 | inline DeletePredicatePB* mutable_delete_pred_pb() { |
330 | 0 | return _rowset_meta_pb.mutable_delete_predicate(); |
331 | 0 | } |
332 | | |
333 | 12.6k | bool is_singleton_delta() const { |
334 | 12.6k | return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version(); |
335 | 12.6k | } |
336 | | |
337 | | // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta. |
338 | | // But, this check behavior may cost a lot of time when it is frequent. |
339 | | // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true, |
340 | | // And next time when we want to check if this rowset is in rowset mata manager's meta, we can |
341 | | // check is_remove_from_rowset_meta() first. |
342 | 314 | void set_remove_from_rowset_meta() { _is_removed_from_rowset_meta = true; } |
343 | | |
344 | 314 | bool is_remove_from_rowset_meta() const { return _is_removed_from_rowset_meta; } |
345 | | |
346 | 32.3k | SegmentsOverlapPB segments_overlap() const { return _rowset_meta_pb.segments_overlap_pb(); } |
347 | | |
348 | 283k | void set_segments_overlap(SegmentsOverlapPB segments_overlap) { |
349 | 283k | _rowset_meta_pb.set_segments_overlap_pb(segments_overlap); |
350 | 283k | } |
351 | | |
352 | 687k | static bool comparator(const RowsetMetaSharedPtr& left, const RowsetMetaSharedPtr& right) { |
353 | 687k | return left->end_version() < right->end_version(); |
354 | 687k | } |
355 | | |
356 | | // return true if segments in this rowset has overlapping data. |
357 | | // this is not same as `segments_overlap()` method. |
358 | | // `segments_overlap()` only return the value of "segments_overlap" field in rowset meta, |
359 | | // but "segments_overlap" may be UNKNOWN. |
360 | | // |
361 | | // Returns true if all of the following conditions are met: |
362 | | // 1. The rowset contains more than one segment. |
363 | | // 2. segments_overlap() is not NONOVERLAPPING (OVERLAP_UNKNOWN, OVERLAPPING, and |
364 | | // NONOVERLAPPING_WITHIN_GROUP are considered overlapping). |
365 | | // 3. The rowset has a singleton version, except row-binlog LMax quick merge rowsets that |
366 | | // explicitly set segments_overlap() to OVERLAPPING. |
367 | 4.89M | bool is_segments_overlapping() const { |
368 | 4.89M | return num_segments() > 1 && segments_overlap() != NONOVERLAPPING && |
369 | 4.89M | (is_singleton_delta() || (is_row_binlog() && segments_overlap() == OVERLAPPING)); |
370 | 4.89M | } |
371 | | |
372 | 43 | bool produced_by_compaction() const { |
373 | 43 | return has_version() && (start_version() < end_version() || |
374 | 43 | (start_version() == end_version() && |
375 | 1 | (segments_overlap() == NONOVERLAPPING || |
376 | 1 | segments_overlap() == NONOVERLAPPING_WITHIN_GROUP))); |
377 | 43 | } |
378 | | |
379 | | // get the compaction score of this rowset. |
380 | | // if segments are overlapping, the score equals to the number of segments, |
381 | | // otherwise, score is 1. |
382 | 2.18M | uint32_t get_compaction_score() const { |
383 | 2.18M | uint32_t score = 0; |
384 | 2.18M | if (!is_segments_overlapping()) { |
385 | 2.17M | score = 1; |
386 | 2.17M | } else { |
387 | 2.36k | auto num_seg = num_segments(); |
388 | 2.36k | DCHECK_GT(num_seg, 0); |
389 | 2.36k | score = cast_set<uint32_t>(num_seg); |
390 | 2.36k | CHECK(score > 0); |
391 | 2.36k | } |
392 | 2.18M | return score; |
393 | 2.18M | } |
394 | | |
395 | 72.1k | uint32_t get_merge_way_num() const { |
396 | 72.1k | uint32_t way_num = 0; |
397 | 72.5k | if (!is_segments_overlapping()) { |
398 | 72.5k | if (num_segments() == 0) { |
399 | 47.8k | way_num = 0; |
400 | 47.8k | } else { |
401 | 24.6k | way_num = 1; |
402 | 24.6k | } |
403 | 18.4E | } else { |
404 | 18.4E | auto num_seg = num_segments(); |
405 | 18.4E | DCHECK_GT(num_seg, 0); |
406 | | |
407 | 18.4E | way_num = cast_set<uint32_t>(num_seg); |
408 | 18.4E | CHECK(way_num > 0); |
409 | 18.4E | } |
410 | 72.1k | return way_num; |
411 | 72.1k | } |
412 | | |
413 | 4.28M | void get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) const { |
414 | 4.28M | for (const KeyBoundsPB& key_range : _rowset_meta_pb.segments_key_bounds()) { |
415 | 4.07M | segments_key_bounds->push_back(key_range); |
416 | 4.07M | } |
417 | 4.28M | } |
418 | | |
419 | 3.43k | auto& get_segments_key_bounds() const { return _rowset_meta_pb.segments_key_bounds(); } |
420 | | |
421 | 5.26M | bool is_segments_key_bounds_truncated() const { |
422 | 5.26M | return _rowset_meta_pb.has_segments_key_bounds_truncated() && |
423 | 5.26M | _rowset_meta_pb.segments_key_bounds_truncated(); |
424 | 5.26M | } |
425 | | |
426 | 300k | void set_segments_key_bounds_truncated(bool truncated) { |
427 | 300k | _rowset_meta_pb.set_segments_key_bounds_truncated(truncated); |
428 | 300k | } |
429 | | |
430 | | // When true, `segments_key_bounds` holds a single aggregated |
431 | | // [rowset_min, rowset_max] entry instead of per-segment bounds. |
432 | 4.26M | bool is_segments_key_bounds_aggregated() const { |
433 | 4.26M | return _rowset_meta_pb.has_segments_key_bounds_aggregated() && |
434 | 4.26M | _rowset_meta_pb.segments_key_bounds_aggregated(); |
435 | 4.26M | } |
436 | | |
437 | 219k | void set_segments_key_bounds_aggregated(bool aggregated) { |
438 | 219k | _rowset_meta_pb.set_segments_key_bounds_aggregated(aggregated); |
439 | 219k | } |
440 | | |
441 | 985k | bool get_first_segment_key_bound(KeyBoundsPB* key_bounds) { |
442 | | // for compatibility, old version has not segment key bounds |
443 | 985k | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
444 | 0 | return false; |
445 | 0 | } |
446 | 985k | *key_bounds = *_rowset_meta_pb.segments_key_bounds().begin(); |
447 | 985k | return true; |
448 | 985k | } |
449 | | |
450 | 654k | bool get_last_segment_key_bound(KeyBoundsPB* key_bounds) { |
451 | 654k | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
452 | 0 | return false; |
453 | 0 | } |
454 | 654k | *key_bounds = *_rowset_meta_pb.segments_key_bounds().rbegin(); |
455 | 654k | return true; |
456 | 654k | } |
457 | | |
458 | | // If `aggregate_into_single` is true, collapse per-segment bounds into a single |
459 | | // [rowset_min, rowset_max] entry and mark this rowset as aggregated. |
460 | | void set_segments_key_bounds(const std::vector<KeyBoundsPB>& segments_key_bounds, |
461 | | bool aggregate_into_single = false); |
462 | | |
463 | 21 | void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) { |
464 | 21 | *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds); |
465 | 21 | set_segments_overlap(OVERLAPPING); |
466 | 21 | } |
467 | | |
468 | 202k | void set_newest_write_timestamp(int64_t timestamp) { |
469 | 202k | _rowset_meta_pb.set_newest_write_timestamp(timestamp); |
470 | 202k | } |
471 | | |
472 | 534k | int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } |
473 | | |
474 | | // for cloud only |
475 | 395 | bool has_visible_ts_ms() const { return _rowset_meta_pb.has_visible_ts_ms(); } |
476 | 388 | int64_t visible_ts_ms() const { return _rowset_meta_pb.visible_ts_ms(); } |
477 | 395 | std::chrono::time_point<std::chrono::system_clock> visible_timestamp() const { |
478 | 395 | using namespace std::chrono; |
479 | 395 | if (has_visible_ts_ms()) { |
480 | 388 | return time_point<system_clock>(milliseconds(visible_ts_ms())); |
481 | 388 | } |
482 | 7 | return system_clock::from_time_t(newest_write_timestamp()); |
483 | 395 | } |
484 | 173k | void set_visible_ts_ms(int64_t visible_ts_ms) { |
485 | 173k | _rowset_meta_pb.set_visible_ts_ms(visible_ts_ms); |
486 | 173k | } |
487 | | |
488 | | void set_tablet_schema(const TabletSchemaSPtr& tablet_schema); |
489 | | void set_tablet_schema(const TabletSchemaPB& tablet_schema); |
490 | | |
491 | 6.02M | const TabletSchemaSPtr& tablet_schema() const { return _schema; } |
492 | | |
493 | 194k | void set_txn_expiration(int64_t expiration) { _rowset_meta_pb.set_txn_expiration(expiration); } |
494 | | |
495 | 196k | void set_compaction_level(int64_t compaction_level) { |
496 | 196k | _rowset_meta_pb.set_compaction_level(compaction_level); |
497 | 196k | } |
498 | | |
499 | 32.2k | int64_t compaction_level() { return _rowset_meta_pb.compaction_level(); } |
500 | | |
501 | | // `seg_file_size` MUST be ordered by rowset segment position. |
502 | | void add_segments_file_size(const std::vector<size_t>& seg_file_size); |
503 | | |
504 | | // Return -1 if segment file size is unknown |
505 | | int64_t segment_file_size_by_pos(size_t pos) const; |
506 | | |
507 | 3.32k | const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); } |
508 | | |
509 | | // Used for partial update, when publish, partial update may add a new rowset and we should update rowset meta |
510 | | void merge_rowset_meta(const RowsetMeta& other); |
511 | | |
512 | | InvertedIndexFileInfo inverted_index_file_info_by_pos(size_t pos) const; |
513 | | |
514 | 260 | const auto& inverted_index_file_info() const { |
515 | 260 | return _rowset_meta_pb.inverted_index_file_info(); |
516 | 260 | } |
517 | | |
518 | | void add_inverted_index_files_info( |
519 | | const std::vector<const InvertedIndexFileInfo*>& idx_file_info); |
520 | | |
521 | | int64_t get_metadata_size() const override; |
522 | | |
523 | | // Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor |
524 | | RowsetMeta(const RowsetMeta&) = delete; |
525 | | RowsetMeta operator=(const RowsetMeta&) = delete; |
526 | | |
527 | | void add_packed_slice_location(const std::string& segment_path, |
528 | | const std::string& packed_file_path, int64_t offset, |
529 | 0 | int64_t size, int64_t packed_file_size) { |
530 | 0 | auto* index_map = _rowset_meta_pb.mutable_packed_slice_locations(); |
531 | 0 | auto& index_pb = (*index_map)[segment_path]; |
532 | 0 | index_pb.set_packed_file_path(packed_file_path); |
533 | 0 | index_pb.set_offset(offset); |
534 | 0 | index_pb.set_size(size); |
535 | 0 | index_pb.set_packed_file_size(packed_file_size); |
536 | 0 | } |
537 | | |
538 | 428 | int32_t schema_version() const { return _rowset_meta_pb.schema_version(); } |
539 | | |
540 | 0 | std::string debug_string() const { return _rowset_meta_pb.ShortDebugString(); } |
541 | | |
542 | | // Pre-set the encryption algorithm to avoid re-entrant get_tablet calls |
543 | | // that can cause SingleFlight deadlock during tablet loading. |
544 | 172k | void set_encryption_algorithm(EncryptionAlgorithmPB algorithm) { |
545 | 172k | _determine_encryption_once.call( |
546 | 172k | [algorithm]() -> Result<EncryptionAlgorithmPB> { return algorithm; }); |
547 | 172k | } |
548 | | |
549 | 3.17M | TsoRange commit_tso() const { |
550 | 3.17M | const auto& commit_tso_pb = _rowset_meta_pb.commit_tso(); |
551 | 3.17M | return {commit_tso_pb.start_tso(), commit_tso_pb.end_tso()}; |
552 | 3.17M | } |
553 | | |
554 | 44 | bool has_commit_tso() const { return _rowset_meta_pb.has_commit_tso(); } |
555 | | |
556 | 9.60k | void set_commit_tso(const TsoRange& commit_tso) { |
557 | 9.60k | auto* commit_tso_pb = _rowset_meta_pb.mutable_commit_tso(); |
558 | 9.60k | commit_tso_pb->set_start_tso(commit_tso.start_tso()); |
559 | 9.60k | commit_tso_pb->set_end_tso(commit_tso.end_tso()); |
560 | 9.60k | } |
561 | | |
562 | 257 | void set_commit_tso(int64_t commit_tso) { set_commit_tso({commit_tso, commit_tso}); } |
563 | | |
564 | 172k | void set_cloud_fields_after_visible(int64_t visible_version, int64_t version_update_time_ms) { |
565 | | // Update rowset meta with correct version and visible_ts |
566 | | // !!ATTENTION!!: this code should be updated if there are more fields |
567 | | // in rowset meta which will be modified in meta-service when commit_txn in the future |
568 | 172k | set_version({visible_version, visible_version}); |
569 | 172k | if (version_update_time_ms > 0) { |
570 | 172k | set_visible_ts_ms(version_update_time_ms); |
571 | 172k | } |
572 | 172k | } |
573 | | |
574 | | private: |
575 | | bool _deserialize_from_pb(std::string_view value); |
576 | | |
577 | | bool _serialize_to_pb(std::string* value); |
578 | | |
579 | | void _init(); |
580 | | |
581 | | void _validate_segment_ids() const; |
582 | | |
583 | | friend bool operator==(const RowsetMeta& a, const RowsetMeta& b); |
584 | | |
585 | 0 | friend bool operator!=(const RowsetMeta& a, const RowsetMeta& b) { return !(a == b); } |
586 | | |
587 | | private: |
588 | | RowsetMetaPB _rowset_meta_pb; |
589 | | TabletSchemaSPtr _schema; |
590 | | Cache::Handle* _handle = nullptr; |
591 | | RowsetId _rowset_id; |
592 | | StorageResource _storage_resource; |
593 | | bool _is_removed_from_rowset_meta = false; |
594 | | DorisCallOnce<Result<EncryptionAlgorithmPB>> _determine_encryption_once; |
595 | | std::atomic<int64_t> _stale_at_s {0}; |
596 | | }; |
597 | | |
598 | | class RowsetSegmentMetaView { |
599 | | public: |
600 | | RowsetSegmentMetaView(const RowsetMeta* meta, size_t pos) |
601 | 3.74M | : _meta(meta), _ref(meta->segment_ref(pos)) {} |
602 | | |
603 | 214k | size_t pos() const { return _ref.pos; } |
604 | 1.96M | int64_t id() const { return _ref.id; } |
605 | 1.74M | RowsetSegmentRef ref() const { return _ref; } |
606 | | |
607 | 203k | int64_t file_size() const { return _meta->segment_file_size_by_pos(pos()); } |
608 | | |
609 | 10.6k | InvertedIndexFileInfo inverted_index_file_info() const { |
610 | 10.6k | return _meta->inverted_index_file_info_by_pos(pos()); |
611 | 10.6k | } |
612 | | |
613 | | bool has_num_rows() const { |
614 | | return cast_set<size_t>(_meta->get_num_segment_rows().size()) > pos(); |
615 | | } |
616 | | |
617 | | int64_t num_rows() const { |
618 | | DORIS_CHECK(has_num_rows()); |
619 | | return _meta->get_num_segment_rows().Get(cast_set<int>(pos())); |
620 | | } |
621 | | |
622 | | bool has_position_key_bounds() const { |
623 | | return !_meta->is_segments_key_bounds_aggregated() && |
624 | | cast_set<size_t>(_meta->get_segments_key_bounds().size()) > pos(); |
625 | | } |
626 | | |
627 | | const KeyBoundsPB& key_bounds() const { |
628 | | DORIS_CHECK(has_position_key_bounds()); |
629 | | return _meta->get_segments_key_bounds().Get(cast_set<int>(pos())); |
630 | | } |
631 | | |
632 | | private: |
633 | | const RowsetMeta* _meta; |
634 | | RowsetSegmentRef _ref; |
635 | | }; |
636 | | |
637 | | class RowsetSegmentMetaRange { |
638 | | public: |
639 | | class Iterator { |
640 | | public: |
641 | | using iterator_category = std::forward_iterator_tag; |
642 | | using value_type = RowsetSegmentMetaView; |
643 | | using difference_type = std::ptrdiff_t; |
644 | | |
645 | 611k | Iterator(const RowsetMeta* meta, size_t pos) : _meta(meta), _pos(pos) {} |
646 | | |
647 | 133k | RowsetSegmentMetaView operator*() const { return {_meta, _pos}; } |
648 | | |
649 | 134k | Iterator& operator++() { |
650 | 134k | ++_pos; |
651 | 134k | return *this; |
652 | 134k | } |
653 | | |
654 | 440k | bool operator==(const Iterator& other) const { |
655 | 441k | return _meta == other._meta && _pos == other._pos; |
656 | 440k | } |
657 | | |
658 | 441k | bool operator!=(const Iterator& other) const { return !(*this == other); } |
659 | | |
660 | | private: |
661 | | const RowsetMeta* _meta; |
662 | | size_t _pos; |
663 | | }; |
664 | | |
665 | 306k | explicit RowsetSegmentMetaRange(const RowsetMeta* meta) : _meta(meta) {} |
666 | | |
667 | 306k | Iterator begin() const { return {_meta, 0}; } |
668 | 306k | Iterator end() const { return {_meta, cast_set<size_t>(_meta->num_segments())}; } |
669 | | |
670 | | private: |
671 | | const RowsetMeta* _meta; |
672 | | }; |
673 | | |
674 | 3.61M | inline RowsetSegmentMetaView RowsetMeta::segment(size_t pos) const { |
675 | 3.61M | return {this, pos}; |
676 | 3.61M | } |
677 | | |
678 | 306k | inline RowsetSegmentMetaRange RowsetMeta::segments() const { |
679 | 306k | return RowsetSegmentMetaRange(this); |
680 | 306k | } |
681 | | |
682 | | } // namespace doris |
683 | | |
684 | | #endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |