be/src/storage/rowset/rowset_meta.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
19 | | #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |
20 | | |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | #include <glog/logging.h> |
23 | | |
24 | | #include <atomic> |
25 | | #include <chrono> |
26 | | #include <cstddef> |
27 | | #include <cstdint> |
28 | | #include <iterator> |
29 | | #include <memory> |
30 | | #include <string> |
31 | | #include <vector> |
32 | | |
33 | | #include "common/cast_set.h" |
34 | | #include "common/check.h" |
35 | | #include "common/config.h" |
36 | | #include "common/status.h" |
37 | | #include "io/fs/encrypted_fs_factory.h" |
38 | | #include "io/fs/file_system.h" |
39 | | #include "runtime/memory/lru_cache_policy.h" |
40 | | #include "storage/metadata_adder.h" |
41 | | #include "storage/olap_common.h" |
42 | | #include "storage/rowset/rowset_fwd.h" |
43 | | #include "storage/rowset/rowset_segment_id.h" |
44 | | #include "storage/storage_policy.h" |
45 | | #include "storage/tablet/tablet_fwd.h" |
46 | | #include "util/once.h" |
47 | | |
48 | | namespace doris { |
49 | | |
50 | | class RowsetSegmentMetaView; |
51 | | class RowsetSegmentMetaRange; |
52 | | |
53 | | class RowsetMeta : public MetadataAdder<RowsetMeta> { |
54 | | public: |
55 | 1.10M | RowsetMeta() = default; |
56 | | ~RowsetMeta(); |
57 | | |
58 | | bool init(std::string_view pb_rowset_meta); |
59 | | |
60 | | bool init(const RowsetMeta* rowset_meta); |
61 | | |
62 | | bool init_from_pb(const RowsetMetaPB& rowset_meta_pb); |
63 | | |
64 | | bool init_from_json(const std::string& json_rowset_meta); |
65 | | |
66 | 6 | bool serialize(std::string* value) { return _serialize_to_pb(value); } |
67 | | |
68 | | bool json_rowset_meta(std::string* json_rowset_meta); |
69 | | |
70 | | // If the rowset is a local rowset, return the global local file system. |
71 | | // Otherwise, return the remote file system corresponding to rowset's resource id. |
72 | | // Note that if the resource id cannot be found for the corresponding remote file system, nullptr will be returned. |
73 | | MOCK_FUNCTION io::FileSystemSPtr fs(); |
74 | | |
75 | | // The bare file system holding this rowset's files. It resolves neither packed files nor |
76 | | // encryption, so it cannot open a segment whose bytes live inside a packed object. Prefer |
77 | | // `fs()`, or `packed_physical_fs()` when the raw bytes are what you are after. |
78 | | io::FileSystemSPtr physical_fs(); |
79 | | |
80 | | // Same as `physical_fs()`, but additionally wrapped with `PackedFileSystem` when this |
81 | | // rowset's files are packed into shared objects, so that segment/index paths still |
82 | | // resolve. Unlike `fs()`, no encryption layer is applied, i.e. reads return the raw |
83 | | // on-disk bytes. Callers that inspect the physical layout of a file (encryption footer, |
84 | | // magic code, ...) must use this instead of `physical_fs()`, otherwise packed files |
85 | | // cannot be opened at all. |
86 | | io::FileSystemSPtr packed_physical_fs(); |
87 | | |
88 | | Result<const StorageResource*> remote_storage_resource(); |
89 | | |
90 | | void set_remote_storage_resource(StorageResource resource); |
91 | | |
92 | 1.74M | const std::string& resource_id() const { return _rowset_meta_pb.resource_id(); } |
93 | | |
94 | 1.25k | void set_resource_id(const std::string& resource_id) { |
95 | 1.25k | _rowset_meta_pb.set_resource_id(resource_id); |
96 | 1.25k | } |
97 | | |
98 | 47.8M | bool is_local() const { return !_rowset_meta_pb.has_resource_id(); } |
99 | | |
100 | | bool has_variant_type_in_schema() const; |
101 | | |
102 | 25.0M | RowsetId rowset_id() const { return _rowset_id; } |
103 | | |
104 | 212k | void set_rowset_id(const RowsetId& rowset_id) { |
105 | | // rowset id is a required field, just set it to 0 |
106 | 212k | _rowset_meta_pb.set_rowset_id(0); |
107 | 212k | _rowset_id = rowset_id; |
108 | 212k | _rowset_meta_pb.set_rowset_id_v2(rowset_id.to_string()); |
109 | 212k | } |
110 | | |
111 | 8.69M | int64_t tablet_id() const { return _rowset_meta_pb.tablet_id(); } |
112 | | |
113 | 212k | void set_tablet_id(int64_t tablet_id) { _rowset_meta_pb.set_tablet_id(tablet_id); } |
114 | | |
115 | 5.11k | int64_t db_id() const { return _rowset_meta_pb.db_id(); } |
116 | | |
117 | 206k | void set_db_id(int64_t db_id) { _rowset_meta_pb.set_db_id(db_id); } |
118 | | |
119 | 4.93k | int64_t table_id() const { return _rowset_meta_pb.table_id(); } |
120 | | |
121 | 207k | void set_table_id(int64_t table_id) { _rowset_meta_pb.set_table_id(table_id); } |
122 | | |
123 | | int64_t index_id() const { return _rowset_meta_pb.index_id(); } |
124 | | |
125 | 206k | void set_index_id(int64_t index_id) { _rowset_meta_pb.set_index_id(index_id); } |
126 | | |
127 | | bool has_inverted_index_storage_format() const { |
128 | | return _rowset_meta_pb.has_inverted_index_storage_format(); |
129 | | } |
130 | | |
131 | | InvertedIndexStorageFormatPB inverted_index_storage_format() const { |
132 | | return _rowset_meta_pb.inverted_index_storage_format(); |
133 | | } |
134 | | |
135 | | void set_inverted_index_storage_format(InvertedIndexStorageFormatPB format); |
136 | | |
137 | 12.9k | TabletUid tablet_uid() const { return _rowset_meta_pb.tablet_uid(); } |
138 | | |
139 | 8.66k | void set_tablet_uid(TabletUid tablet_uid) { |
140 | 8.66k | *(_rowset_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto(); |
141 | 8.66k | } |
142 | | |
143 | 680k | int64_t txn_id() const { return _rowset_meta_pb.txn_id(); } |
144 | | |
145 | 204k | void set_txn_id(int64_t txn_id) { _rowset_meta_pb.set_txn_id(txn_id); } |
146 | | |
147 | 1.41k | int32_t tablet_schema_hash() const { return _rowset_meta_pb.tablet_schema_hash(); } |
148 | | |
149 | 207k | void set_tablet_schema_hash(int32_t tablet_schema_hash) { |
150 | 207k | _rowset_meta_pb.set_tablet_schema_hash(tablet_schema_hash); |
151 | 207k | } |
152 | | |
153 | 2.52k | void mark_row_binlog() { _rowset_meta_pb.set_is_row_binlog(true); } |
154 | | |
155 | 155k | bool is_row_binlog() const { |
156 | 155k | return _rowset_meta_pb.has_is_row_binlog() && _rowset_meta_pb.is_row_binlog(); |
157 | 155k | } |
158 | | |
159 | 2.00M | RowsetTypePB rowset_type() const { return _rowset_meta_pb.rowset_type(); } |
160 | | |
161 | 210k | void set_rowset_type(RowsetTypePB rowset_type) { _rowset_meta_pb.set_rowset_type(rowset_type); } |
162 | | |
163 | 590k | RowsetStatePB rowset_state() const { return _rowset_meta_pb.rowset_state(); } |
164 | | |
165 | 419k | void set_rowset_state(RowsetStatePB rowset_state) { |
166 | 419k | _rowset_meta_pb.set_rowset_state(rowset_state); |
167 | 419k | } |
168 | | |
169 | 49.6M | Version version() const { |
170 | 49.6M | return {_rowset_meta_pb.start_version(), _rowset_meta_pb.end_version()}; |
171 | 49.6M | } |
172 | | |
173 | 259k | void set_version(Version version) { |
174 | 259k | _rowset_meta_pb.set_start_version(version.first); |
175 | 259k | _rowset_meta_pb.set_end_version(version.second); |
176 | 259k | } |
177 | | |
178 | 1.13M | bool has_version() const { |
179 | 1.13M | return _rowset_meta_pb.has_start_version() && _rowset_meta_pb.has_end_version(); |
180 | 1.13M | } |
181 | | |
182 | 4.95M | int64_t start_version() const { return _rowset_meta_pb.start_version(); } |
183 | | |
184 | 7.87M | int64_t end_version() const { return _rowset_meta_pb.end_version(); } |
185 | | |
186 | 9.07M | int64_t num_rows() const { return _rowset_meta_pb.num_rows(); } |
187 | | |
188 | 238k | void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); } |
189 | | |
190 | 231k | void set_num_segment_rows(const std::vector<uint32_t>& num_segment_rows) { |
191 | 231k | _rowset_meta_pb.mutable_num_segment_rows()->Assign(num_segment_rows.cbegin(), |
192 | 231k | num_segment_rows.cend()); |
193 | 231k | } |
194 | | |
195 | 137 | void get_num_segment_rows(std::vector<uint32_t>* num_segment_rows) const { |
196 | 137 | num_segment_rows->assign(_rowset_meta_pb.num_segment_rows().cbegin(), |
197 | 137 | _rowset_meta_pb.num_segment_rows().cend()); |
198 | 137 | } |
199 | | |
200 | 169k | auto& get_num_segment_rows() const { return _rowset_meta_pb.num_segment_rows(); } |
201 | | |
202 | 14 | void set_segment_group_sizes(const std::vector<int32_t>& segment_group_sizes) { |
203 | 14 | DORIS_CHECK_GT(segment_group_sizes.size(), 1); |
204 | 14 | int64_t segment_count = 0; |
205 | 45 | for (const auto group_size : segment_group_sizes) { |
206 | 45 | DORIS_CHECK_GT(group_size, 0); |
207 | 45 | segment_count += group_size; |
208 | 45 | } |
209 | 14 | DORIS_CHECK_EQ(segment_count, num_segments()); |
210 | 14 | _rowset_meta_pb.mutable_segment_group_sizes()->Assign(segment_group_sizes.cbegin(), |
211 | 14 | segment_group_sizes.cend()); |
212 | 14 | } |
213 | | |
214 | | void clear_segment_group_sizes() { _rowset_meta_pb.clear_segment_group_sizes(); } |
215 | | |
216 | 34 | const auto& segment_group_sizes() const { return _rowset_meta_pb.segment_group_sizes(); } |
217 | | |
218 | 10.4M | int64_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); } |
219 | | |
220 | 250k | void set_total_disk_size(int64_t total_disk_size) { |
221 | 250k | _rowset_meta_pb.set_total_disk_size(total_disk_size); |
222 | 250k | } |
223 | | |
224 | 9.04M | int64_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); } |
225 | | |
226 | 238k | void set_data_disk_size(int64_t data_disk_size) { |
227 | 238k | _rowset_meta_pb.set_data_disk_size(data_disk_size); |
228 | 238k | } |
229 | | |
230 | 2.52M | int64_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); } |
231 | | |
232 | 238k | void set_index_disk_size(int64_t index_disk_size) { |
233 | 238k | _rowset_meta_pb.set_index_disk_size(index_disk_size); |
234 | 238k | } |
235 | | |
236 | 0 | void zone_maps(std::vector<::doris::ZoneMap>* zone_maps) { |
237 | 0 | for (const ::doris::ZoneMap& zone_map : _rowset_meta_pb.zone_maps()) { |
238 | 0 | zone_maps->push_back(zone_map); |
239 | 0 | } |
240 | 0 | } |
241 | | |
242 | 0 | void set_zone_maps(const std::vector<::doris::ZoneMap>& zone_maps) { |
243 | 0 | for (const ::doris::ZoneMap& zone_map : zone_maps) { |
244 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
245 | 0 | *new_zone_map = zone_map; |
246 | 0 | } |
247 | 0 | } |
248 | | |
249 | 0 | void add_zone_map(const ::doris::ZoneMap& zone_map) { |
250 | 0 | ::doris::ZoneMap* new_zone_map = _rowset_meta_pb.add_zone_maps(); |
251 | 0 | *new_zone_map = zone_map; |
252 | 0 | } |
253 | | |
254 | 5.12M | bool has_delete_predicate() const { return _rowset_meta_pb.has_delete_predicate(); } |
255 | | |
256 | 8.99k | const DeletePredicatePB& delete_predicate() const { return _rowset_meta_pb.delete_predicate(); } |
257 | | |
258 | 0 | DeletePredicatePB* mutable_delete_predicate() { |
259 | 0 | return _rowset_meta_pb.mutable_delete_predicate(); |
260 | 0 | } |
261 | | |
262 | 3.24k | void set_delete_predicate(DeletePredicatePB delete_predicate) { |
263 | 3.24k | DeletePredicatePB* new_delete_condition = _rowset_meta_pb.mutable_delete_predicate(); |
264 | 3.24k | *new_delete_condition = std::move(delete_predicate); |
265 | 3.24k | } |
266 | | |
267 | 8.29k | bool empty() const { return _rowset_meta_pb.empty(); } |
268 | | |
269 | 232k | void set_empty(bool empty) { _rowset_meta_pb.set_empty(empty); } |
270 | | |
271 | 136 | PUniqueId load_id() const { return _rowset_meta_pb.load_id(); } |
272 | | |
273 | 187k | void set_load_id(PUniqueId load_id) { |
274 | 187k | PUniqueId* new_load_id = _rowset_meta_pb.mutable_load_id(); |
275 | 187k | new_load_id->set_hi(load_id.hi()); |
276 | 187k | new_load_id->set_lo(load_id.lo()); |
277 | 187k | } |
278 | | |
279 | 206k | void set_job_id(const std::string& job_id) { _rowset_meta_pb.set_job_id(job_id); } |
280 | | |
281 | 0 | const std::string& job_id() const { return _rowset_meta_pb.job_id(); } |
282 | | |
283 | 0 | bool delete_flag() const { return _rowset_meta_pb.delete_flag(); } |
284 | | |
285 | 130k | int64_t creation_time() const { return _rowset_meta_pb.creation_time(); } |
286 | | |
287 | 233k | void set_creation_time(int64_t creation_time) { |
288 | 233k | return _rowset_meta_pb.set_creation_time(creation_time); |
289 | 233k | } |
290 | | |
291 | 528k | int64_t stale_at() const { |
292 | 528k | int64_t stale_time = _stale_at_s.load(); |
293 | 528k | return stale_time > 0 ? stale_time : _rowset_meta_pb.creation_time(); |
294 | 528k | } |
295 | | |
296 | 12.4k | bool has_stale_at() const { return _stale_at_s.load() > 0; } |
297 | | |
298 | 89.8k | void set_stale_at(int64_t stale_at) { _stale_at_s.store(stale_at); } |
299 | | |
300 | 11.5k | int64_t partition_id() const { return _rowset_meta_pb.partition_id(); } |
301 | | |
302 | 207k | void set_partition_id(int64_t partition_id) { |
303 | 207k | return _rowset_meta_pb.set_partition_id(partition_id); |
304 | 207k | } |
305 | | |
306 | 31.6M | int64_t num_segments() const { |
307 | 31.6M | DCHECK(_rowset_meta_pb.segment_ids_size() == 0 || |
308 | 31.6M | _rowset_meta_pb.segment_ids_size() == _rowset_meta_pb.num_segments()); |
309 | 31.6M | return _rowset_meta_pb.num_segments(); |
310 | 31.6M | } |
311 | | |
312 | 274k | void set_num_segments(int64_t num_segments) { _rowset_meta_pb.set_num_segments(num_segments); } |
313 | | |
314 | 4.81M | bool has_segment_ids() const { return _rowset_meta_pb.segment_ids_size() > 0; } |
315 | | |
316 | 291k | const auto& segment_ids() const { return _rowset_meta_pb.segment_ids(); } |
317 | | |
318 | | void set_segment_ids(const std::vector<int64_t>& segment_ids); |
319 | | |
320 | 3.85M | int64_t segment_id(size_t pos) const { |
321 | 3.85M | DORIS_CHECK_LT(pos, cast_set<size_t>(num_segments())); |
322 | 3.85M | return has_segment_ids() ? _rowset_meta_pb.segment_ids(cast_set<int>(pos)) |
323 | 3.85M | : cast_set<int64_t>(pos); |
324 | 3.85M | } |
325 | | |
326 | 3.85M | RowsetSegmentRef segment_ref(size_t pos) const { return {pos, segment_id(pos)}; } |
327 | | |
328 | | RowsetSegmentMetaView segment(size_t pos) const; |
329 | | |
330 | | RowsetSegmentMetaRange segments() const; |
331 | | |
332 | | size_t position_of(int64_t seg_id) const; |
333 | | |
334 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
335 | | void to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema = false) const; |
336 | | |
337 | | // Convert to RowsetMetaPB, skip_schema is only used by cloud to separate schema from rowset meta. |
338 | | RowsetMetaPB get_rowset_pb(bool skip_schema = false) const; |
339 | | |
340 | 70 | inline DeletePredicatePB* mutable_delete_pred_pb() { |
341 | 70 | return _rowset_meta_pb.mutable_delete_predicate(); |
342 | 70 | } |
343 | | |
344 | 123k | bool is_singleton_delta() const { |
345 | 123k | return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version(); |
346 | 123k | } |
347 | | |
348 | | // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta. |
349 | | // But, this check behavior may cost a lot of time when it is frequent. |
350 | | // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true, |
351 | | // And next time when we want to check if this rowset is in rowset mata manager's meta, we can |
352 | | // check is_remove_from_rowset_meta() first. |
353 | 4.07k | void set_remove_from_rowset_meta() { _is_removed_from_rowset_meta = true; } |
354 | | |
355 | 4.07k | bool is_remove_from_rowset_meta() const { return _is_removed_from_rowset_meta; } |
356 | | |
357 | 34.0k | SegmentsOverlapPB segments_overlap() const { return _rowset_meta_pb.segments_overlap_pb(); } |
358 | | |
359 | 298k | void set_segments_overlap(SegmentsOverlapPB segments_overlap) { |
360 | 298k | _rowset_meta_pb.set_segments_overlap_pb(segments_overlap); |
361 | 298k | } |
362 | | |
363 | 621k | static bool comparator(const RowsetMetaSharedPtr& left, const RowsetMetaSharedPtr& right) { |
364 | 621k | return left->end_version() < right->end_version(); |
365 | 621k | } |
366 | | |
367 | | // return true if segments in this rowset has overlapping data. |
368 | | // this is not same as `segments_overlap()` method. |
369 | | // `segments_overlap()` only return the value of "segments_overlap" field in rowset meta, |
370 | | // but "segments_overlap" may be UNKNOWN. |
371 | | // |
372 | | // Returns true if all of the following conditions are met: |
373 | | // 1. The rowset contains more than one segment. |
374 | | // 2. segments_overlap() is not NONOVERLAPPING (OVERLAP_UNKNOWN, OVERLAPPING, and |
375 | | // NONOVERLAPPING_WITHIN_GROUP are considered overlapping). |
376 | | // 3. The rowset has a singleton version, except row-binlog LMax quick merge rowsets that |
377 | | // explicitly set segments_overlap() to OVERLAPPING. |
378 | 5.12M | bool is_segments_overlapping() const { |
379 | 5.12M | return num_segments() > 1 && segments_overlap() != NONOVERLAPPING && |
380 | 5.12M | (is_singleton_delta() || (is_row_binlog() && segments_overlap() == OVERLAPPING)); |
381 | 5.12M | } |
382 | | |
383 | 52 | bool produced_by_compaction() const { |
384 | 52 | return has_version() && (start_version() < end_version() || |
385 | 51 | (start_version() == end_version() && |
386 | 1 | (segments_overlap() == NONOVERLAPPING || |
387 | 1 | segments_overlap() == NONOVERLAPPING_WITHIN_GROUP))); |
388 | 52 | } |
389 | | |
390 | | // get the compaction score of this rowset. |
391 | | // if segments are overlapping, the score equals to the number of segments, |
392 | | // otherwise, score is 1. |
393 | 2.29M | uint32_t get_compaction_score() const { |
394 | 2.29M | uint32_t score = 0; |
395 | 2.29M | if (!is_segments_overlapping()) { |
396 | 2.28M | score = 1; |
397 | 2.28M | } else { |
398 | 3.31k | auto num_seg = num_segments(); |
399 | 3.31k | DCHECK_GT(num_seg, 0); |
400 | 3.31k | score = cast_set<uint32_t>(num_seg); |
401 | 3.31k | CHECK(score > 0); |
402 | 3.31k | } |
403 | 2.29M | return score; |
404 | 2.29M | } |
405 | | |
406 | 76.9k | uint32_t get_merge_way_num() const { |
407 | 76.9k | uint32_t way_num = 0; |
408 | 77.3k | if (!is_segments_overlapping()) { |
409 | 77.3k | if (num_segments() == 0) { |
410 | 50.2k | way_num = 0; |
411 | 50.2k | } else { |
412 | 27.0k | way_num = 1; |
413 | 27.0k | } |
414 | 18.4E | } else { |
415 | 18.4E | auto num_seg = num_segments(); |
416 | 18.4E | DCHECK_GT(num_seg, 0); |
417 | | |
418 | 18.4E | way_num = cast_set<uint32_t>(num_seg); |
419 | 18.4E | CHECK(way_num > 0); |
420 | 18.4E | } |
421 | 76.9k | return way_num; |
422 | 76.9k | } |
423 | | |
424 | 260 | void get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) const { |
425 | 275 | for (const KeyBoundsPB& key_range : _rowset_meta_pb.segments_key_bounds()) { |
426 | 275 | segments_key_bounds->push_back(key_range); |
427 | 275 | } |
428 | 260 | } |
429 | | |
430 | 4.19M | auto& get_segments_key_bounds() const { return _rowset_meta_pb.segments_key_bounds(); } |
431 | | |
432 | 5.28M | bool is_segments_key_bounds_truncated() const { |
433 | 5.28M | return _rowset_meta_pb.has_segments_key_bounds_truncated() && |
434 | 5.28M | _rowset_meta_pb.segments_key_bounds_truncated(); |
435 | 5.28M | } |
436 | | |
437 | 237k | void set_segments_key_bounds_truncated(bool truncated) { |
438 | 237k | _rowset_meta_pb.set_segments_key_bounds_truncated(truncated); |
439 | 237k | } |
440 | | |
441 | | // When true, `segments_key_bounds` holds a single aggregated |
442 | | // [rowset_min, rowset_max] entry instead of per-segment bounds. |
443 | 4.17M | bool is_segments_key_bounds_aggregated() const { |
444 | 4.17M | return _rowset_meta_pb.has_segments_key_bounds_aggregated() && |
445 | 4.17M | _rowset_meta_pb.segments_key_bounds_aggregated(); |
446 | 4.17M | } |
447 | | |
448 | 231k | void set_segments_key_bounds_aggregated(bool aggregated) { |
449 | 231k | _rowset_meta_pb.set_segments_key_bounds_aggregated(aggregated); |
450 | 231k | } |
451 | | |
452 | 1.07M | bool get_first_segment_key_bound(KeyBoundsPB* key_bounds) { |
453 | | // for compatibility, old version has not segment key bounds |
454 | 1.07M | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
455 | 0 | return false; |
456 | 0 | } |
457 | 1.07M | *key_bounds = *_rowset_meta_pb.segments_key_bounds().begin(); |
458 | 1.07M | return true; |
459 | 1.07M | } |
460 | | |
461 | 814k | bool get_last_segment_key_bound(KeyBoundsPB* key_bounds) { |
462 | 814k | if (_rowset_meta_pb.segments_key_bounds_size() == 0) { |
463 | 0 | return false; |
464 | 0 | } |
465 | 814k | *key_bounds = *_rowset_meta_pb.segments_key_bounds().rbegin(); |
466 | 814k | return true; |
467 | 814k | } |
468 | | |
469 | | // If `aggregate_into_single` is true, collapse per-segment bounds into a single |
470 | | // [rowset_min, rowset_max] entry and mark this rowset as aggregated. |
471 | | void set_segments_key_bounds(const std::vector<KeyBoundsPB>& segments_key_bounds, |
472 | | bool aggregate_into_single = false); |
473 | | |
474 | 34 | void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) { |
475 | 34 | *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds); |
476 | 34 | set_segments_overlap(OVERLAPPING); |
477 | 34 | } |
478 | | |
479 | 211k | void set_newest_write_timestamp(int64_t timestamp) { |
480 | 211k | _rowset_meta_pb.set_newest_write_timestamp(timestamp); |
481 | 211k | } |
482 | | |
483 | 641k | int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } |
484 | | |
485 | | // for cloud only |
486 | 290 | bool has_visible_ts_ms() const { return _rowset_meta_pb.has_visible_ts_ms(); } |
487 | 288 | int64_t visible_ts_ms() const { return _rowset_meta_pb.visible_ts_ms(); } |
488 | 290 | std::chrono::time_point<std::chrono::system_clock> visible_timestamp() const { |
489 | 290 | using namespace std::chrono; |
490 | 290 | if (has_visible_ts_ms()) { |
491 | 288 | return time_point<system_clock>(milliseconds(visible_ts_ms())); |
492 | 288 | } |
493 | 2 | return system_clock::from_time_t(newest_write_timestamp()); |
494 | 290 | } |
495 | 177k | void set_visible_ts_ms(int64_t visible_ts_ms) { |
496 | 177k | _rowset_meta_pb.set_visible_ts_ms(visible_ts_ms); |
497 | 177k | } |
498 | | |
499 | | void set_tablet_schema(const TabletSchemaSPtr& tablet_schema); |
500 | | void set_tablet_schema(const TabletSchemaPB& tablet_schema); |
501 | | |
502 | 6.47M | const TabletSchemaSPtr& tablet_schema() const { return _schema; } |
503 | | |
504 | 203k | void set_txn_expiration(int64_t expiration) { _rowset_meta_pb.set_txn_expiration(expiration); } |
505 | | |
506 | 206k | void set_compaction_level(int64_t compaction_level) { |
507 | 206k | _rowset_meta_pb.set_compaction_level(compaction_level); |
508 | 206k | } |
509 | | |
510 | 226k | int64_t compaction_level() { return _rowset_meta_pb.compaction_level(); } |
511 | | |
512 | | // `seg_file_size` MUST be ordered by rowset segment position. |
513 | | void add_segments_file_size(const std::vector<size_t>& seg_file_size); |
514 | | |
515 | | // Return -1 if segment file size is unknown |
516 | | int64_t segment_file_size_by_pos(size_t pos) const; |
517 | | |
518 | 5.33k | const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); } |
519 | | |
520 | | // Used for partial update, when publish, partial update may add a new rowset and we should update rowset meta |
521 | | void merge_rowset_meta(const RowsetMeta& other); |
522 | | |
523 | | InvertedIndexFileInfo inverted_index_file_info_by_pos(size_t pos) const; |
524 | | |
525 | 260 | const auto& inverted_index_file_info() const { |
526 | 260 | return _rowset_meta_pb.inverted_index_file_info(); |
527 | 260 | } |
528 | | |
529 | | void add_inverted_index_files_info( |
530 | | const std::vector<const InvertedIndexFileInfo*>& idx_file_info); |
531 | | |
532 | | int64_t get_metadata_size() const override; |
533 | | |
534 | | // Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor |
535 | | RowsetMeta(const RowsetMeta&) = delete; |
536 | | RowsetMeta operator=(const RowsetMeta&) = delete; |
537 | | |
538 | | void add_packed_slice_location(const std::string& segment_path, |
539 | | const std::string& packed_file_path, int64_t offset, |
540 | 3 | int64_t size, int64_t packed_file_size) { |
541 | 3 | auto* index_map = _rowset_meta_pb.mutable_packed_slice_locations(); |
542 | 3 | auto& index_pb = (*index_map)[segment_path]; |
543 | 3 | index_pb.set_packed_file_path(packed_file_path); |
544 | 3 | index_pb.set_offset(offset); |
545 | 3 | index_pb.set_size(size); |
546 | 3 | index_pb.set_packed_file_size(packed_file_size); |
547 | 3 | } |
548 | | |
549 | 690 | int32_t schema_version() const { return _rowset_meta_pb.schema_version(); } |
550 | | |
551 | 0 | std::string debug_string() const { return _rowset_meta_pb.ShortDebugString(); } |
552 | | |
553 | | // Pre-set the encryption algorithm to avoid re-entrant get_tablet calls |
554 | | // that can cause SingleFlight deadlock during tablet loading. |
555 | 176k | void set_encryption_algorithm(EncryptionAlgorithmPB algorithm) { |
556 | 176k | _determine_encryption_once.call( |
557 | 176k | [algorithm]() -> Result<EncryptionAlgorithmPB> { return algorithm; }); |
558 | 176k | } |
559 | | |
560 | 3.12M | TsoRange commit_tso() const { |
561 | 3.12M | const auto& commit_tso_pb = _rowset_meta_pb.commit_tso(); |
562 | 3.12M | return {commit_tso_pb.start_tso(), commit_tso_pb.end_tso()}; |
563 | 3.12M | } |
564 | | |
565 | 47 | bool has_commit_tso() const { return _rowset_meta_pb.has_commit_tso(); } |
566 | | |
567 | 10.6k | void set_commit_tso(const TsoRange& commit_tso) { |
568 | 10.6k | auto* commit_tso_pb = _rowset_meta_pb.mutable_commit_tso(); |
569 | 10.6k | commit_tso_pb->set_start_tso(commit_tso.start_tso()); |
570 | 10.6k | commit_tso_pb->set_end_tso(commit_tso.end_tso()); |
571 | 10.6k | } |
572 | | |
573 | 467 | void set_commit_tso(int64_t commit_tso) { set_commit_tso({commit_tso, commit_tso}); } |
574 | | |
575 | 176k | void set_cloud_fields_after_visible(int64_t visible_version, int64_t version_update_time_ms) { |
576 | | // Update rowset meta with correct version and visible_ts |
577 | | // !!ATTENTION!!: this code should be updated if there are more fields |
578 | | // in rowset meta which will be modified in meta-service when commit_txn in the future |
579 | 176k | set_version({visible_version, visible_version}); |
580 | 176k | if (version_update_time_ms > 0) { |
581 | 176k | set_visible_ts_ms(version_update_time_ms); |
582 | 176k | } |
583 | 176k | } |
584 | | |
585 | | private: |
586 | | // Wraps `fs` with `PackedFileSystem` if this rowset has packed slice locations, |
587 | | // otherwise returns `fs` unchanged. |
588 | | io::FileSystemSPtr _wrap_packed_fs(io::FileSystemSPtr fs); |
589 | | |
590 | | bool _deserialize_from_pb(std::string_view value); |
591 | | |
592 | | bool _serialize_to_pb(std::string* value); |
593 | | |
594 | | void _init(); |
595 | | |
596 | | void _validate_segment_ids() const; |
597 | | |
598 | | friend bool operator==(const RowsetMeta& a, const RowsetMeta& b); |
599 | | |
600 | 0 | friend bool operator!=(const RowsetMeta& a, const RowsetMeta& b) { return !(a == b); } |
601 | | |
602 | | private: |
603 | | RowsetMetaPB _rowset_meta_pb; |
604 | | TabletSchemaSPtr _schema; |
605 | | Cache::Handle* _handle = nullptr; |
606 | | RowsetId _rowset_id; |
607 | | StorageResource _storage_resource; |
608 | | bool _is_removed_from_rowset_meta = false; |
609 | | DorisCallOnce<Result<EncryptionAlgorithmPB>> _determine_encryption_once; |
610 | | std::atomic<int64_t> _stale_at_s {0}; |
611 | | }; |
612 | | |
613 | | class RowsetSegmentMetaView { |
614 | | public: |
615 | | RowsetSegmentMetaView(const RowsetMeta* meta, size_t pos) |
616 | 3.83M | : _meta(meta), _ref(meta->segment_ref(pos)) {} |
617 | | |
618 | 226k | size_t pos() const { return _ref.pos; } |
619 | 2.10M | int64_t id() const { return _ref.id; } |
620 | 1.68M | RowsetSegmentRef ref() const { return _ref; } |
621 | | |
622 | 215k | int64_t file_size() const { return _meta->segment_file_size_by_pos(pos()); } |
623 | | |
624 | 10.7k | InvertedIndexFileInfo inverted_index_file_info() const { |
625 | 10.7k | return _meta->inverted_index_file_info_by_pos(pos()); |
626 | 10.7k | } |
627 | | |
628 | | bool has_num_rows() const { |
629 | | return cast_set<size_t>(_meta->get_num_segment_rows().size()) > pos(); |
630 | | } |
631 | | |
632 | | int64_t num_rows() const { |
633 | | DORIS_CHECK(has_num_rows()); |
634 | | return _meta->get_num_segment_rows().Get(cast_set<int>(pos())); |
635 | | } |
636 | | |
637 | | bool has_position_key_bounds() const { |
638 | | return !_meta->is_segments_key_bounds_aggregated() && |
639 | | cast_set<size_t>(_meta->get_segments_key_bounds().size()) > pos(); |
640 | | } |
641 | | |
642 | | const KeyBoundsPB& key_bounds() const { |
643 | | DORIS_CHECK(has_position_key_bounds()); |
644 | | return _meta->get_segments_key_bounds().Get(cast_set<int>(pos())); |
645 | | } |
646 | | |
647 | | private: |
648 | | const RowsetMeta* _meta; |
649 | | RowsetSegmentRef _ref; |
650 | | }; |
651 | | |
652 | | class RowsetSegmentMetaRange { |
653 | | public: |
654 | | class Iterator { |
655 | | public: |
656 | | using iterator_category = std::forward_iterator_tag; |
657 | | using value_type = RowsetSegmentMetaView; |
658 | | using difference_type = std::ptrdiff_t; |
659 | | |
660 | 693k | Iterator(const RowsetMeta* meta, size_t pos) : _meta(meta), _pos(pos) {} |
661 | | |
662 | 150k | RowsetSegmentMetaView operator*() const { return {_meta, _pos}; } |
663 | | |
664 | 150k | Iterator& operator++() { |
665 | 150k | ++_pos; |
666 | 150k | return *this; |
667 | 150k | } |
668 | | |
669 | 498k | bool operator==(const Iterator& other) const { |
670 | 498k | return _meta == other._meta && _pos == other._pos; |
671 | 498k | } |
672 | | |
673 | 498k | bool operator!=(const Iterator& other) const { return !(*this == other); } |
674 | | |
675 | | private: |
676 | | const RowsetMeta* _meta; |
677 | | size_t _pos; |
678 | | }; |
679 | | |
680 | 347k | explicit RowsetSegmentMetaRange(const RowsetMeta* meta) : _meta(meta) {} |
681 | | |
682 | 347k | Iterator begin() const { return {_meta, 0}; } |
683 | 348k | Iterator end() const { return {_meta, cast_set<size_t>(_meta->num_segments())}; } |
684 | | |
685 | | private: |
686 | | const RowsetMeta* _meta; |
687 | | }; |
688 | | |
689 | 3.68M | inline RowsetSegmentMetaView RowsetMeta::segment(size_t pos) const { |
690 | 3.68M | return {this, pos}; |
691 | 3.68M | } |
692 | | |
693 | 347k | inline RowsetSegmentMetaRange RowsetMeta::segments() const { |
694 | 347k | return RowsetSegmentMetaRange(this); |
695 | 347k | } |
696 | | |
697 | | } // namespace doris |
698 | | |
699 | | #endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H |