be/src/storage/tablet/tablet_meta.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <gen_cpp/AgentService_types.h> |
21 | | #include <gen_cpp/FrontendService_types.h> |
22 | | #include <gen_cpp/olap_file.pb.h> |
23 | | #include <stdint.h> |
24 | | |
25 | | #include <atomic> |
26 | | #include <cstddef> |
27 | | #include <limits> |
28 | | #include <map> |
29 | | #include <memory> |
30 | | #include <mutex> |
31 | | #include <optional> |
32 | | #include <ostream> |
33 | | #include <roaring/roaring.hh> |
34 | | #include <shared_mutex> |
35 | | #include <string> |
36 | | #include <tuple> |
37 | | #include <unordered_map> |
38 | | #include <utility> |
39 | | #include <vector> |
40 | | |
41 | | #include "common/logging.h" |
42 | | #include "common/status.h" |
43 | | #include "io/fs/file_system.h" |
44 | | #include "runtime/memory/lru_cache_policy.h" |
45 | | #include "storage/binlog_config.h" |
46 | | #include "storage/metadata_adder.h" |
47 | | #include "storage/olap_common.h" |
48 | | #include "storage/rowset/rowset_meta.h" |
49 | | #include "storage/tablet/tablet_schema.h" |
50 | | #include "util/lru_cache.h" |
51 | | #include "util/uid_util.h" |
52 | | |
53 | | namespace json2pb { |
54 | | #include "common/compile_check_begin.h" |
55 | | struct Pb2JsonOptions; |
56 | | } // namespace json2pb |
57 | | |
58 | | namespace doris { |
59 | | class TColumn; |
60 | | |
61 | | // Lifecycle states that a Tablet can be in. Legal state transitions for a |
62 | | // Tablet object: |
63 | | // |
64 | | // NOTREADY -> RUNNING -> TOMBSTONED -> STOPPED -> SHUTDOWN |
65 | | // | | | ^^^ |
66 | | // | | +----------++| |
67 | | // | +------------------------+| |
68 | | // +-------------------------------------+ |
69 | | |
70 | | enum TabletState { |
71 | | // Tablet is under alter table, rollup, clone |
72 | | TABLET_NOTREADY, |
73 | | |
74 | | TABLET_RUNNING, |
75 | | |
76 | | // Tablet integrity has been violated, such as missing versions. |
77 | | // In this state, tablet will not accept any incoming request. |
78 | | // Report this state to FE, scheduling BE to drop tablet. |
79 | | TABLET_TOMBSTONED, |
80 | | |
81 | | // Tablet is shutting down, files in disk still remained. |
82 | | TABLET_STOPPED, |
83 | | |
84 | | // Files have been removed, tablet has been shutdown completely. |
85 | | TABLET_SHUTDOWN |
86 | | }; |
87 | | |
88 | | class DataDir; |
89 | | class TabletMeta; |
90 | | class DeleteBitmap; |
91 | | class TBinlogConfig; |
92 | | |
93 | | // Class encapsulates meta of tablet. |
94 | | // The concurrency control is handled in Tablet Class, not in this class. |
95 | | class TabletMeta : public MetadataAdder<TabletMeta> { |
96 | | public: |
97 | | static TabletMetaSharedPtr create( |
98 | | const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t shard_id, |
99 | | uint32_t next_unique_id, |
100 | | const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id); |
101 | | |
102 | | TabletMeta(); |
103 | | ~TabletMeta() override; |
104 | | TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id, |
105 | | int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema, |
106 | | uint32_t next_unique_id, |
107 | | const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id, |
108 | | TabletUid tablet_uid, TTabletType::type tabletType, |
109 | | TCompressionType::type compression_type, int64_t storage_policy_id = 0, |
110 | | bool enable_unique_key_merge_on_write = false, |
111 | | std::optional<TBinlogConfig> binlog_config = {}, |
112 | | std::string compaction_policy = "size_based", |
113 | | int64_t time_series_compaction_goal_size_mbytes = 1024, |
114 | | int64_t time_series_compaction_file_count_threshold = 2000, |
115 | | int64_t time_series_compaction_time_threshold_seconds = 3600, |
116 | | int64_t time_series_compaction_empty_rowsets_threshold = 5, |
117 | | int64_t time_series_compaction_level_threshold = 1, |
118 | | TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format = |
119 | | TInvertedIndexFileStorageFormat::V2, |
120 | | TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT, |
121 | | TStorageFormat::type storage_format = TStorageFormat::V2, |
122 | | int32_t vertical_compaction_num_columns_per_group = 5); |
123 | | // If need add a filed in TableMeta, filed init copy in copy construct function |
124 | | TabletMeta(const TabletMeta& tablet_meta); |
125 | | TabletMeta(TabletMeta&& tablet_meta) = delete; |
126 | | |
127 | | // UT |
128 | | #ifdef BE_TEST |
129 | | TabletMeta(TabletSchemaSPtr tablet_schema) : _schema(tablet_schema) {} |
130 | | #endif |
131 | | |
132 | | // Function create_from_file is used to be compatible with previous tablet_meta. |
133 | | // Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb. |
134 | | Status create_from_file(const std::string& file_path); |
135 | | // Used to create tablet meta from memory buffer. |
136 | | Status create_from_buffer(const uint8_t* buffer, size_t buffer_size); |
137 | | static Status load_from_file(const std::string& file_path, TabletMetaPB* tablet_meta_pb); |
138 | | Status save(const std::string& file_path); |
139 | | Status save_as_json(const std::string& file_path); |
140 | | static Status save(const std::string& file_path, const TabletMetaPB& tablet_meta_pb); |
141 | | static std::string construct_header_file_path(const std::string& schema_hash_path, |
142 | | int64_t tablet_id); |
143 | | Status save_meta(DataDir* data_dir); |
144 | | |
145 | | void serialize(std::string* meta_binary); |
146 | | Status deserialize(std::string_view meta_binary); |
147 | | void init_from_pb(const TabletMetaPB& tablet_meta_pb); |
148 | | |
149 | | void to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_meta); |
150 | | void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options); |
151 | 366k | size_t tablet_columns_num() const { return _schema->num_columns(); } |
152 | | |
153 | 0 | TabletTypePB tablet_type() const { return _tablet_type; } |
154 | | TabletUid tablet_uid() const; |
155 | 206 | void set_tablet_uid(TabletUid uid) { _tablet_uid = uid; } |
156 | | int64_t table_id() const; |
157 | | int64_t index_id() const; |
158 | | int64_t partition_id() const; |
159 | | int64_t tablet_id() const; |
160 | | int64_t replica_id() const; |
161 | 0 | void set_replica_id(int64_t replica_id) { _replica_id = replica_id; } |
162 | | int32_t schema_hash() const; |
163 | | int32_t shard_id() const; |
164 | | void set_shard_id(int32_t shard_id); |
165 | | int64_t creation_time() const; |
166 | | void set_creation_time(int64_t creation_time); |
167 | | int64_t cumulative_layer_point() const; |
168 | | void set_cumulative_layer_point(int64_t new_point); |
169 | | |
170 | | size_t num_rows() const; |
171 | | // Disk space occupied by tablet, contain local and remote. |
172 | | size_t tablet_footprint() const; |
173 | | // Local disk space occupied by tablet. |
174 | | size_t tablet_local_size() const; |
175 | | // Remote disk space occupied by tablet. |
176 | | size_t tablet_remote_size() const; |
177 | | |
178 | | size_t tablet_local_index_size() const; |
179 | | size_t tablet_local_segment_size() const; |
180 | | size_t tablet_remote_index_size() const; |
181 | | size_t tablet_remote_segment_size() const; |
182 | | |
183 | | size_t version_count() const; |
184 | | size_t stale_version_count() const; |
185 | | size_t version_count_cross_with_range(const Version& range) const; |
186 | | Version max_version() const; |
187 | | |
188 | | TabletState tablet_state() const; |
189 | | void set_tablet_state(TabletState state); |
190 | | |
191 | | bool in_restore_mode() const; |
192 | | void set_in_restore_mode(bool in_restore_mode); |
193 | | |
194 | | const TabletSchemaSPtr& tablet_schema() const; |
195 | | |
196 | | TabletSchema* mutable_tablet_schema(); |
197 | | |
198 | | const RowsetMetaMapContainer& all_rs_metas() const; |
199 | | RowsetMetaMapContainer& all_mutable_rs_metas(); |
200 | | Status add_rs_meta(const RowsetMetaSharedPtr& rs_meta); |
201 | | void delete_rs_meta_by_version(const Version& version, |
202 | | std::vector<RowsetMetaSharedPtr>* deleted_rs_metas); |
203 | | // If same_version is true, the rowset in "to_delete" will not be added |
204 | | // to _stale_rs_meta, but to be deleted from rs_meta directly. |
205 | | void modify_rs_metas(const std::vector<RowsetMetaSharedPtr>& to_add, |
206 | | const std::vector<RowsetMetaSharedPtr>& to_delete, |
207 | | bool same_version = false); |
208 | | void revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas); |
209 | | void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap); |
210 | | |
211 | | const RowsetMetaMapContainer& all_stale_rs_metas() const; |
212 | | RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version) const; |
213 | | void delete_stale_rs_meta_by_version(const Version& version); |
214 | | RowsetMetaSharedPtr acquire_stale_rs_meta_by_version(const Version& version) const; |
215 | | |
216 | | Status set_partition_id(int64_t partition_id); |
217 | | |
218 | 22.8k | RowsetTypePB preferred_rowset_type() const { return _preferred_rowset_type; } |
219 | | |
220 | 6.08k | void set_preferred_rowset_type(RowsetTypePB preferred_rowset_type) { |
221 | 6.08k | _preferred_rowset_type = preferred_rowset_type; |
222 | 6.08k | } |
223 | | |
224 | | // used for after tablet cloned to clear stale rowset |
225 | | void clear_stale_rowset(); |
226 | | |
227 | | void clear_rowsets(); |
228 | | |
229 | | // MUST hold EXCLUSIVE `_meta_lock` in belonged Tablet |
230 | | // `to_add` MUST NOT have overlapped version with `_rs_metas` in tablet meta. |
231 | | void add_rowsets_unchecked(const std::vector<RowsetSharedPtr>& to_add); |
232 | | |
233 | | bool all_beta() const; |
234 | | |
235 | 2.01M | int64_t storage_policy_id() const { return _storage_policy_id; } |
236 | | |
237 | 4 | void set_storage_policy_id(int64_t id) { |
238 | 4 | VLOG_NOTICE << "set tablet_id : " << _table_id << " storage policy from " |
239 | 4 | << _storage_policy_id << " to " << id; |
240 | 4 | _storage_policy_id = id; |
241 | 4 | } |
242 | | |
243 | 3.23M | UniqueId cooldown_meta_id() const { return _cooldown_meta_id; } |
244 | 5 | void set_cooldown_meta_id(UniqueId uid) { _cooldown_meta_id = uid; } |
245 | | |
246 | | static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, |
247 | | ColumnPB* column); |
248 | | |
249 | 3.01M | DeleteBitmapPtr delete_bitmap_ptr() { return _delete_bitmap; } |
250 | 1.00M | DeleteBitmap& delete_bitmap() { return *_delete_bitmap; } |
251 | | void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version); |
252 | | |
253 | 6.67M | bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } |
254 | | #ifdef BE_TEST |
255 | | void set_enable_unique_key_merge_on_write(bool value) { |
256 | | _enable_unique_key_merge_on_write = value; |
257 | | } |
258 | | #endif |
259 | | // TODO(Drogon): thread safety |
260 | 14.4k | const BinlogConfig& binlog_config() const { return _binlog_config; } |
261 | 0 | void set_binlog_config(BinlogConfig binlog_config) { |
262 | 0 | _binlog_config = std::move(binlog_config); |
263 | 0 | } |
264 | | |
265 | 13 | void set_compaction_policy(std::string compaction_policy) { |
266 | 13 | _compaction_policy = compaction_policy; |
267 | 13 | } |
268 | 518M | std::string compaction_policy() const { return _compaction_policy; } |
269 | 12 | void set_time_series_compaction_goal_size_mbytes(int64_t goal_size_mbytes) { |
270 | 12 | _time_series_compaction_goal_size_mbytes = goal_size_mbytes; |
271 | 12 | } |
272 | 174k | int64_t time_series_compaction_goal_size_mbytes() const { |
273 | 174k | return _time_series_compaction_goal_size_mbytes; |
274 | 174k | } |
275 | 12 | void set_time_series_compaction_file_count_threshold(int64_t file_count_threshold) { |
276 | 12 | _time_series_compaction_file_count_threshold = file_count_threshold; |
277 | 12 | } |
278 | 174k | int64_t time_series_compaction_file_count_threshold() const { |
279 | 174k | return _time_series_compaction_file_count_threshold; |
280 | 174k | } |
281 | 13 | void set_time_series_compaction_time_threshold_seconds(int64_t time_threshold) { |
282 | 13 | _time_series_compaction_time_threshold_seconds = time_threshold; |
283 | 13 | } |
284 | 174k | int64_t time_series_compaction_time_threshold_seconds() const { |
285 | 174k | return _time_series_compaction_time_threshold_seconds; |
286 | 174k | } |
287 | 0 | void set_time_series_compaction_empty_rowsets_threshold(int64_t empty_rowsets_threshold) { |
288 | 0 | _time_series_compaction_empty_rowsets_threshold = empty_rowsets_threshold; |
289 | 0 | } |
290 | 174k | int64_t time_series_compaction_empty_rowsets_threshold() const { |
291 | 174k | return _time_series_compaction_empty_rowsets_threshold; |
292 | 174k | } |
293 | 0 | void set_time_series_compaction_level_threshold(int64_t level_threshold) { |
294 | 0 | _time_series_compaction_level_threshold = level_threshold; |
295 | 0 | } |
296 | 174k | int64_t time_series_compaction_level_threshold() const { |
297 | 174k | return _time_series_compaction_level_threshold; |
298 | 174k | } |
299 | | |
300 | 0 | void set_vertical_compaction_num_columns_per_group(int32_t num) { |
301 | 0 | _vertical_compaction_num_columns_per_group = num; |
302 | 0 | } |
303 | 185k | int32_t vertical_compaction_num_columns_per_group() const { |
304 | 185k | return _vertical_compaction_num_columns_per_group; |
305 | 185k | } |
306 | | |
307 | 1.71M | int64_t ttl_seconds() const { |
308 | 1.71M | std::shared_lock rlock(_meta_lock); |
309 | 1.71M | return _ttl_seconds; |
310 | 1.71M | } |
311 | | |
312 | | void set_ttl_seconds(int64_t ttl_seconds) { |
313 | | std::lock_guard wlock(_meta_lock); |
314 | | _ttl_seconds = ttl_seconds; |
315 | | } |
316 | | |
317 | 14.9k | int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; } |
318 | | |
319 | 347k | EncryptionAlgorithmPB encryption_algorithm() const { return _encryption_algorithm; } |
320 | | |
321 | | private: |
322 | | Status _save_meta(DataDir* data_dir); |
323 | | void _check_mow_rowset_cache_version_size(size_t rowset_cache_version_size); |
324 | | |
325 | | // _del_predicates is ignored to compare. |
326 | | friend bool operator==(const TabletMeta& a, const TabletMeta& b); |
327 | | friend bool operator!=(const TabletMeta& a, const TabletMeta& b); |
328 | | |
329 | | private: |
330 | | int64_t _table_id = 0; |
331 | | int64_t _index_id = 0; |
332 | | int64_t _partition_id = 0; |
333 | | int64_t _tablet_id = 0; |
334 | | int64_t _replica_id = 0; |
335 | | int32_t _schema_hash = 0; |
336 | | int32_t _shard_id = 0; |
337 | | int64_t _creation_time = 0; |
338 | | int64_t _cumulative_layer_point = 0; |
339 | | TabletUid _tablet_uid; |
340 | | TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK; |
341 | | |
342 | | TabletState _tablet_state = TABLET_NOTREADY; |
343 | | // the reference of _schema may use in tablet, so here need keep |
344 | | // the lifetime of tablemeta and _schema is same with tablet |
345 | | TabletSchemaSPtr _schema; |
346 | | Cache::Handle* _handle = nullptr; |
347 | | |
348 | | RowsetMetaMapContainer _rs_metas; |
349 | | // This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted. |
350 | | // These stale rowsets meta are been removed when rowsets' pathVersion is expired, |
351 | | // this policy is judged and computed by TimestampedVersionTracker. |
352 | | RowsetMetaMapContainer _stale_rs_metas; |
353 | | bool _in_restore_mode = false; |
354 | | RowsetTypePB _preferred_rowset_type = BETA_ROWSET; |
355 | | |
356 | | // meta for cooldown |
357 | | int64_t _storage_policy_id = 0; // <= 0 means no storage policy |
358 | | UniqueId _cooldown_meta_id; |
359 | | |
360 | | // For unique key data model, the feature Merge-on-Write will leverage a primary |
361 | | // key index and a delete-bitmap to mark duplicate keys as deleted in load stage, |
362 | | // which can avoid the merging cost in read stage, and accelerate the aggregation |
363 | | // query performance significantly. |
364 | | bool _enable_unique_key_merge_on_write = false; |
365 | | std::shared_ptr<DeleteBitmap> _delete_bitmap; |
366 | | |
367 | | // binlog config |
368 | | BinlogConfig _binlog_config {}; |
369 | | |
370 | | // meta for compaction |
371 | | std::string _compaction_policy; |
372 | | int64_t _time_series_compaction_goal_size_mbytes = 0; |
373 | | int64_t _time_series_compaction_file_count_threshold = 0; |
374 | | int64_t _time_series_compaction_time_threshold_seconds = 0; |
375 | | int64_t _time_series_compaction_empty_rowsets_threshold = 0; |
376 | | int64_t _time_series_compaction_level_threshold = 0; |
377 | | int32_t _vertical_compaction_num_columns_per_group = 5; |
378 | | |
379 | | int64_t _avg_rs_meta_serialize_size = 0; |
380 | | |
381 | | // cloud |
382 | | int64_t _ttl_seconds = 0; |
383 | | |
384 | | EncryptionAlgorithmPB _encryption_algorithm = PLAINTEXT; |
385 | | |
386 | | // Persisted storage format for this tablet (e.g. V2, V3). Used to derive |
387 | | // schema-level defaults such as external ColumnMeta usage. |
388 | | TStorageFormat::type _storage_format = TStorageFormat::V2; |
389 | | |
390 | | mutable std::shared_mutex _meta_lock; |
391 | | }; |
392 | | |
393 | | class DeleteBitmapAggCache : public LRUCachePolicy { |
394 | | public: |
395 | | DeleteBitmapAggCache(size_t capacity); |
396 | | |
397 | | static DeleteBitmapAggCache* instance(); |
398 | | |
399 | | static DeleteBitmapAggCache* create_instance(size_t capacity); |
400 | | |
401 | | DeleteBitmap snapshot(int64_t tablet_id); |
402 | | |
403 | | class Value : public LRUCacheValueBase { |
404 | | public: |
405 | | roaring::Roaring bitmap; |
406 | | }; |
407 | | }; |
408 | | |
409 | | /** |
410 | | * Wraps multiple bitmaps for recording rows (row id) that are deleted or |
411 | | * overwritten. For now, it's only used when unique key merge-on-write property |
412 | | * enabled. |
413 | | * |
414 | | * RowsetId and SegmentId are for locating segment, Version here is a single |
415 | | * uint32_t means that at which "version" of the load causes the delete or |
416 | | * overwrite. |
417 | | * |
418 | | * The start and end version of a load is the same, it's ok and straightforward |
419 | | * to use a single uint32_t. |
420 | | * |
421 | | * e.g. |
422 | | * There is a key "key1" in rowset id 1, version [1,1], segment id 1, row id 1. |
423 | | * A new load also contains "key1", the rowset id 2, version [2,2], segment id 1 |
424 | | * the delete bitmap will be `{1,1,2} -> 1`, which means the "row id 1" in |
425 | | * "rowset id 1, segment id 1" is deleted/overitten by some loads at "version 2" |
426 | | */ |
427 | | class DeleteBitmap { |
428 | | public: |
429 | | mutable std::shared_mutex lock; |
430 | | using SegmentId = uint32_t; |
431 | | using Version = uint64_t; |
432 | | using BitmapKey = std::tuple<RowsetId, SegmentId, Version>; |
433 | | std::map<BitmapKey, roaring::Roaring> delete_bitmap; // Ordered map |
434 | | constexpr static inline uint32_t INVALID_SEGMENT_ID = std::numeric_limits<uint32_t>::max() - 1; |
435 | | constexpr static inline uint32_t ROWSET_SENTINEL_MARK = |
436 | | std::numeric_limits<uint32_t>::max() - 1; |
437 | | |
438 | | // When a delete bitmap is merged into tablet's delete bitmap, the version of entries in the delete bitmap |
439 | | // will be replaced to the correspoding correct version. So before we finally merge a delete bitmap into |
440 | | // tablet's delete bitmap we can use arbitary version number in BitmapKey. Here we define some version numbers |
441 | | // for specific usage during this periods to avoid conflicts |
442 | | constexpr static inline uint64_t TEMP_VERSION_COMMON = 0; |
443 | | |
444 | | /** |
445 | | * |
446 | | * @param tablet_id the tablet which this delete bitmap associates with |
447 | | */ |
448 | | DeleteBitmap(int64_t tablet_id); |
449 | | |
450 | | /** |
451 | | * Copy c-tor for making delete bitmap snapshot on read path |
452 | | */ |
453 | | DeleteBitmap(const DeleteBitmap& r); |
454 | | DeleteBitmap& operator=(const DeleteBitmap& r); |
455 | | /** |
456 | | * Move c-tor for making delete bitmap snapshot on read path |
457 | | */ |
458 | | DeleteBitmap(DeleteBitmap&& r) noexcept; |
459 | | DeleteBitmap& operator=(DeleteBitmap&& r) noexcept; |
460 | | |
461 | | static DeleteBitmap from_pb(const DeleteBitmapPB& pb, int64_t tablet_id); |
462 | | |
463 | | DeleteBitmapPB to_pb(); |
464 | | |
465 | | /** |
466 | | * Makes a snapshot of delete bitmap, read lock will be acquired in this |
467 | | * process |
468 | | */ |
469 | | DeleteBitmap snapshot() const; |
470 | | |
471 | | /** |
472 | | * Makes a snapshot of delete bitmap on given version, read lock will be |
473 | | * acquired temporary in this process |
474 | | */ |
475 | | DeleteBitmap snapshot(Version version) const; |
476 | | |
477 | | /** |
478 | | * Marks the specific row deleted |
479 | | */ |
480 | | void add(const BitmapKey& bmk, uint32_t row_id); |
481 | | |
482 | | /** |
483 | | * Clears the deletetion mark specific row |
484 | | * |
485 | | * @return non-zero if the associated delete bitmap does not exist |
486 | | */ |
487 | | int remove(const BitmapKey& bmk, uint32_t row_id); |
488 | | |
489 | | /** |
490 | | * Clears bitmaps in range [lower_key, upper_key) |
491 | | */ |
492 | | void remove(const BitmapKey& lower_key, const BitmapKey& upper_key); |
493 | | void remove(const std::vector<std::tuple<BitmapKey, BitmapKey>>& key_ranges); |
494 | | |
495 | | /** |
496 | | * Checks if the given row is marked deleted |
497 | | * |
498 | | * @return true if marked deleted |
499 | | */ |
500 | | bool contains(const BitmapKey& bmk, uint32_t row_id) const; |
501 | | |
502 | | /** |
503 | | * Checks if this delete bitmap is empty |
504 | | * |
505 | | * @return true if empty |
506 | | */ |
507 | | bool empty() const; |
508 | | |
509 | | /** |
510 | | * return the total cardinality of the Delete Bitmap |
511 | | */ |
512 | | uint64_t cardinality() const; |
513 | | |
514 | | /** |
515 | | * return the total size of the Delete Bitmap(after serialized) |
516 | | */ |
517 | | |
518 | | uint64_t get_size() const; |
519 | | |
520 | | /** |
521 | | * Sets the bitmap of specific segment, it's may be insertion or replacement |
522 | | * |
523 | | * @return 1 if the insertion took place, 0 if the assignment took place |
524 | | */ |
525 | | int set(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap); |
526 | | |
527 | | /** |
528 | | * Gets a copy of specific delete bmk |
529 | | * |
530 | | * @param segment_delete_bitmap output param |
531 | | * @return non-zero if the associated delete bitmap does not exist |
532 | | */ |
533 | | int get(const BitmapKey& bmk, roaring::Roaring* segment_delete_bitmap) const; |
534 | | |
535 | | /** |
536 | | * Gets reference to a specific delete map, DO NOT use this function on a |
537 | | * mutable DeleteBitmap object |
538 | | * @return nullptr if the given bitmap does not exist |
539 | | */ |
540 | | const roaring::Roaring* get(const BitmapKey& bmk) const; |
541 | | |
542 | | /** |
543 | | * Gets subset of delete_bitmap with given range [start, end) |
544 | | * |
545 | | * @parma start start |
546 | | * @parma end end |
547 | | * @parma subset_delete_map output param |
548 | | */ |
549 | | void subset(const BitmapKey& start, const BitmapKey& end, |
550 | | DeleteBitmap* subset_delete_map) const; |
551 | | void subset(std::vector<std::pair<RowsetId, int64_t>>& rowset_ids, int64_t start_version, |
552 | | int64_t end_version, DeleteBitmap* subset_delete_map) const; |
553 | | |
554 | | /** |
555 | | * Gets subset of delete_bitmap of the input rowsets |
556 | | * with given version range [start_version, end_version] and agg to end_version, |
557 | | * then merge to subset_delete_map |
558 | | */ |
559 | | void subset_and_agg(std::vector<std::pair<RowsetId, int64_t>>& rowset_ids, |
560 | | int64_t start_version, int64_t end_version, |
561 | | DeleteBitmap* subset_delete_map) const; |
562 | | |
563 | | /** |
564 | | * Gets count of delete_bitmap with given range [start, end) |
565 | | * |
566 | | * @parma start start |
567 | | * @parma end end |
568 | | */ |
569 | | size_t get_count_with_range(const BitmapKey& start, const BitmapKey& end) const; |
570 | | |
571 | | /** |
572 | | * Merges the given segment delete bitmap into *this |
573 | | * |
574 | | * @param bmk |
575 | | * @param segment_delete_bitmap |
576 | | */ |
577 | | void merge(const BitmapKey& bmk, const roaring::Roaring& segment_delete_bitmap); |
578 | | |
579 | | /** |
580 | | * Merges the given delete bitmap into *this |
581 | | * |
582 | | * @param other |
583 | | */ |
584 | | void merge(const DeleteBitmap& other); |
585 | | |
586 | | /** |
587 | | * Checks if the given row is marked deleted in bitmap with the condition: |
588 | | * all the bitmaps that |
589 | | * RowsetId and SegmentId are the same as the given ones, |
590 | | * and Version <= the given Version |
591 | | * |
592 | | * Note: aggregation cache may be used. |
593 | | * |
594 | | * @return true if marked deleted |
595 | | */ |
596 | | bool contains_agg(const BitmapKey& bitmap, uint32_t row_id) const; |
597 | | |
598 | | bool contains_agg_with_cache_if_eligible(const BitmapKey& bmk, uint32_t row_id) const; |
599 | | /** |
600 | | * Gets aggregated delete_bitmap on rowset_id and version, the same effect: |
601 | | * `select sum(roaring::Roaring) where RowsetId=rowset_id and SegmentId=seg_id and Version <= version` |
602 | | * |
603 | | * @return shared_ptr to a bitmap, which may be empty |
604 | | */ |
605 | | std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const; |
606 | | std::shared_ptr<roaring::Roaring> get_agg_without_cache(const BitmapKey& bmk, |
607 | | const int64_t start_version = 0) const; |
608 | | |
609 | | void remove_sentinel_marks(); |
610 | | |
611 | | uint64_t get_delete_bitmap_count(); |
612 | | |
613 | | void traverse_rowset_and_version( |
614 | | const std::function<int(const RowsetId& rowsetId, int64_t version)>& func) const; |
615 | | |
616 | | bool has_calculated_for_multi_segments(const RowsetId& rowset_id) const; |
617 | | |
618 | | // return the size of the map |
619 | | size_t remove_rowset_cache_version(const RowsetId& rowset_id); |
620 | | |
621 | | void clear_rowset_cache_version(); |
622 | | |
623 | | std::set<std::string> get_rowset_cache_version(); |
624 | | |
625 | | DeleteBitmap agg_cache_snapshot(); |
626 | | |
627 | | void set_tablet_id(int64_t tablet_id); |
628 | | |
629 | | /** |
630 | | * Calculate diffset with given `key_set`. All entries with keys contained in this delete bitmap but not |
631 | | * in given key_set will be added to the output delete bitmap. |
632 | | * |
633 | | * @return Deletebitmap containning all entries in diffset |
634 | | */ |
635 | | DeleteBitmap diffset(const std::set<BitmapKey>& key_set) const; |
636 | | |
637 | | private: |
638 | | DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk) const; |
639 | | |
640 | | int64_t _tablet_id; |
641 | | mutable std::shared_mutex _rowset_cache_version_lock; |
642 | | mutable std::map<RowsetId, std::map<SegmentId, Version>> _rowset_cache_version; |
643 | | }; |
644 | | |
645 | 2.85M | inline TabletUid TabletMeta::tablet_uid() const { |
646 | 2.85M | return _tablet_uid; |
647 | 2.85M | } |
648 | | |
649 | 1.06M | inline int64_t TabletMeta::table_id() const { |
650 | 1.06M | return _table_id; |
651 | 1.06M | } |
652 | | |
653 | 1.30M | inline int64_t TabletMeta::index_id() const { |
654 | 1.30M | return _index_id; |
655 | 1.30M | } |
656 | | |
657 | 4.75M | inline int64_t TabletMeta::partition_id() const { |
658 | 4.75M | return _partition_id; |
659 | 4.75M | } |
660 | | |
661 | 1.82G | inline int64_t TabletMeta::tablet_id() const { |
662 | 1.82G | return _tablet_id; |
663 | 1.82G | } |
664 | | |
665 | 1.47M | inline int64_t TabletMeta::replica_id() const { |
666 | 1.47M | return _replica_id; |
667 | 1.47M | } |
668 | | |
669 | 2.16M | inline int32_t TabletMeta::schema_hash() const { |
670 | 2.16M | return _schema_hash; |
671 | 2.16M | } |
672 | | |
673 | 430k | inline int32_t TabletMeta::shard_id() const { |
674 | 430k | return _shard_id; |
675 | 430k | } |
676 | | |
677 | 189 | inline void TabletMeta::set_shard_id(int32_t shard_id) { |
678 | 189 | _shard_id = shard_id; |
679 | 189 | } |
680 | | |
681 | 234k | inline int64_t TabletMeta::creation_time() const { |
682 | 234k | return _creation_time; |
683 | 234k | } |
684 | | |
685 | | inline void TabletMeta::set_creation_time(int64_t creation_time) { |
686 | | _creation_time = creation_time; |
687 | | } |
688 | | |
689 | 67.9k | inline int64_t TabletMeta::cumulative_layer_point() const { |
690 | 67.9k | return _cumulative_layer_point; |
691 | 67.9k | } |
692 | | |
693 | 0 | inline void TabletMeta::set_cumulative_layer_point(int64_t new_point) { |
694 | 0 | _cumulative_layer_point = new_point; |
695 | 0 | } |
696 | | |
697 | 1.25M | inline size_t TabletMeta::num_rows() const { |
698 | 1.25M | size_t num_rows = 0; |
699 | 2.28M | for (const auto& [_, rs] : _rs_metas) { |
700 | 2.28M | num_rows += rs->num_rows(); |
701 | 2.28M | } |
702 | 1.25M | return num_rows; |
703 | 1.25M | } |
704 | | |
705 | 22.0k | inline size_t TabletMeta::tablet_footprint() const { |
706 | 22.0k | size_t total_size = 0; |
707 | 61.6k | for (const auto& [_, rs] : _rs_metas) { |
708 | 61.6k | total_size += rs->total_disk_size(); |
709 | 61.6k | } |
710 | 22.0k | return total_size; |
711 | 22.0k | } |
712 | | |
713 | 4.15M | inline size_t TabletMeta::tablet_local_size() const { |
714 | 4.15M | size_t total_size = 0; |
715 | 7.57M | for (const auto& [_, rs] : _rs_metas) { |
716 | 7.57M | if (rs->is_local()) { |
717 | 7.54M | total_size += rs->total_disk_size(); |
718 | 7.54M | } |
719 | 7.57M | } |
720 | 4.15M | return total_size; |
721 | 4.15M | } |
722 | | |
723 | 4.15M | inline size_t TabletMeta::tablet_remote_size() const { |
724 | 4.15M | size_t total_size = 0; |
725 | 7.57M | for (const auto& [_, rs] : _rs_metas) { |
726 | 7.57M | if (!rs->is_local()) { |
727 | 29.5k | total_size += rs->total_disk_size(); |
728 | 29.5k | } |
729 | 7.57M | } |
730 | 4.15M | return total_size; |
731 | 4.15M | } |
732 | | |
733 | 1.25M | inline size_t TabletMeta::tablet_local_index_size() const { |
734 | 1.25M | size_t total_size = 0; |
735 | 2.28M | for (const auto& [_, rs] : _rs_metas) { |
736 | 2.28M | if (rs->is_local()) { |
737 | 2.28M | total_size += rs->index_disk_size(); |
738 | 2.28M | } |
739 | 2.28M | } |
740 | 1.25M | return total_size; |
741 | 1.25M | } |
742 | | |
743 | 1.25M | inline size_t TabletMeta::tablet_local_segment_size() const { |
744 | 1.25M | size_t total_size = 0; |
745 | 2.28M | for (const auto& [_, rs] : _rs_metas) { |
746 | 2.28M | if (rs->is_local()) { |
747 | 2.28M | total_size += rs->data_disk_size(); |
748 | 2.28M | } |
749 | 2.28M | } |
750 | 1.25M | return total_size; |
751 | 1.25M | } |
752 | | |
753 | 1.25M | inline size_t TabletMeta::tablet_remote_index_size() const { |
754 | 1.25M | size_t total_size = 0; |
755 | 2.28M | for (const auto& [_, rs] : _rs_metas) { |
756 | 2.28M | if (!rs->is_local()) { |
757 | 14 | total_size += rs->index_disk_size(); |
758 | 14 | } |
759 | 2.28M | } |
760 | 1.25M | return total_size; |
761 | 1.25M | } |
762 | | |
763 | 1.25M | inline size_t TabletMeta::tablet_remote_segment_size() const { |
764 | 1.25M | size_t total_size = 0; |
765 | 2.28M | for (const auto& [_, rs] : _rs_metas) { |
766 | 2.28M | if (!rs->is_local()) { |
767 | 14 | total_size += rs->data_disk_size(); |
768 | 14 | } |
769 | 2.28M | } |
770 | 1.25M | return total_size; |
771 | 1.25M | } |
772 | | |
773 | 9.67M | inline size_t TabletMeta::version_count() const { |
774 | 9.67M | return _rs_metas.size(); |
775 | 9.67M | } |
776 | | |
777 | 14.9k | inline size_t TabletMeta::stale_version_count() const { |
778 | 14.9k | return _rs_metas.size(); |
779 | 14.9k | } |
780 | | |
781 | 489M | inline TabletState TabletMeta::tablet_state() const { |
782 | 489M | return _tablet_state; |
783 | 489M | } |
784 | | |
785 | 22.8k | inline void TabletMeta::set_tablet_state(TabletState state) { |
786 | 22.8k | _tablet_state = state; |
787 | 22.8k | } |
788 | | |
789 | 68.0k | inline bool TabletMeta::in_restore_mode() const { |
790 | 68.0k | return _in_restore_mode; |
791 | 68.0k | } |
792 | | |
793 | 0 | inline void TabletMeta::set_in_restore_mode(bool in_restore_mode) { |
794 | 0 | _in_restore_mode = in_restore_mode; |
795 | 0 | } |
796 | | |
797 | 784M | inline const TabletSchemaSPtr& TabletMeta::tablet_schema() const { |
798 | 784M | return _schema; |
799 | 784M | } |
800 | | |
801 | 5 | inline TabletSchema* TabletMeta::mutable_tablet_schema() { |
802 | 5 | return _schema.get(); |
803 | 5 | } |
804 | | |
805 | 11.5M | inline const RowsetMetaMapContainer& TabletMeta::all_rs_metas() const { |
806 | 11.5M | return _rs_metas; |
807 | 11.5M | } |
808 | | |
809 | 336k | inline RowsetMetaMapContainer& TabletMeta::all_mutable_rs_metas() { |
810 | 336k | return _rs_metas; |
811 | 336k | } |
812 | | |
813 | 854k | inline const RowsetMetaMapContainer& TabletMeta::all_stale_rs_metas() const { |
814 | 854k | return _stale_rs_metas; |
815 | 854k | } |
816 | | |
817 | 0 | inline bool TabletMeta::all_beta() const { |
818 | 0 | for (const auto& [_, rs] : _rs_metas) { |
819 | 0 | if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) { |
820 | 0 | return false; |
821 | 0 | } |
822 | 0 | } |
823 | 0 | for (const auto& [_, rs] : _stale_rs_metas) { |
824 | 0 | if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) { |
825 | 0 | return false; |
826 | 0 | } |
827 | 0 | } |
828 | 0 | return true; |
829 | 0 | } |
830 | | |
831 | | std::string tablet_state_name(TabletState state); |
832 | | |
833 | | // Only for unit test now. |
834 | | bool operator==(const TabletMeta& a, const TabletMeta& b); |
835 | | bool operator!=(const TabletMeta& a, const TabletMeta& b); |
836 | | |
837 | | #include "common/compile_check_end.h" |
838 | | } // namespace doris |