be/src/cloud/cloud_txn_delete_bitmap_cache.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <mutex> |
21 | | |
22 | | #include "storage/olap_common.h" |
23 | | #include "storage/partial_update_info.h" |
24 | | #include "storage/rowset/rowset.h" |
25 | | #include "storage/tablet/tablet_meta.h" |
26 | | #include "storage/txn/txn_manager.h" |
27 | | #include "util/countdown_latch.h" |
28 | | #include "util/lru_cache.h" |
29 | | |
30 | | namespace doris { |
31 | | |
32 | | // Record transaction related delete bitmaps using a lru cache. |
33 | | class CloudTxnDeleteBitmapCache : public LRUCachePolicy { |
34 | | public: |
35 | | CloudTxnDeleteBitmapCache(size_t size_in_bytes); |
36 | | |
37 | | ~CloudTxnDeleteBitmapCache() override; |
38 | | |
39 | | Status init(); |
40 | | |
41 | | Status get_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, |
42 | | RowsetSharedPtr* rowset, DeleteBitmapPtr* delete_bitmap, |
43 | | RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration, |
44 | | std::shared_ptr<PartialUpdateInfo>* partial_update_info, |
45 | | std::shared_ptr<PublishStatus>* publish_status, |
46 | | TxnPublishInfo* previous_publish_info); |
47 | | |
48 | | void set_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, |
49 | | DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids, |
50 | | RowsetSharedPtr rowset, int64_t txn_expirationm, |
51 | | std::shared_ptr<PartialUpdateInfo> partial_update_info); |
52 | | |
53 | | Status update_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, |
54 | | DeleteBitmapPtr delete_bitmap, |
55 | | const RowsetIdUnorderedSet& rowset_ids, |
56 | | PublishStatus publish_status, TxnPublishInfo publish_info = {}); |
57 | | |
58 | | void remove_expired_tablet_txn_info(); |
59 | | |
60 | | void remove_unused_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id); |
61 | | |
62 | | // Mark a rowset as empty/skipped (lightweight marker, no rowset stored) |
63 | | // Used for empty rowsets when skip_writing_empty_rowset_metadata is enabled |
64 | | void mark_empty_rowset(TTransactionId txn_id, int64_t tablet_id, int64_t txn_expiration); |
65 | | |
66 | | // Check if this is a known empty/skipped rowset |
67 | | // Returns true if was marked as empty rowset |
68 | | // Note: Does not remove the marker, as CalcDeleteBitmapTask may retry. |
69 | | // Cleanup is handled by expiration-based removal in remove_expired_tablet_txn_info() |
70 | | bool is_empty_rowset(TTransactionId txn_id, int64_t tablet_id); |
71 | | |
72 | | // !!!ATTENTION!!!: the delete bitmap stored in CloudTxnDeleteBitmapCache contains sentinel marks, |
73 | | // and the version in BitmapKey is DeleteBitmap::TEMP_VERSION_COMMON. |
74 | | // when using delete bitmap from this cache, the caller should manually remove these marks if don't need it |
75 | | // and should replace versions in BitmapKey by the correct version |
76 | | Status get_delete_bitmap(TTransactionId transaction_id, int64_t tablet_id, |
77 | | DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, |
78 | | std::shared_ptr<PublishStatus>* publish_status); |
79 | | |
80 | | private: |
81 | | void _clean_thread_callback(); |
82 | | |
83 | | class DeleteBitmapCacheValue : public LRUCacheValueBase { |
84 | | public: |
85 | | DeleteBitmapPtr delete_bitmap; |
86 | | // records rowsets calc in commit txn |
87 | | RowsetIdUnorderedSet rowset_ids; |
88 | | |
89 | | DeleteBitmapCacheValue(DeleteBitmapPtr delete_bitmap_, const RowsetIdUnorderedSet& ids_) |
90 | 0 | : delete_bitmap(std::move(delete_bitmap_)), rowset_ids(ids_) {} |
91 | | }; |
92 | | |
93 | | struct TxnKey { |
94 | | TTransactionId txn_id; |
95 | | int64_t tablet_id; |
96 | | TxnKey(TTransactionId txn_id_, int64_t tablet_id_) |
97 | 0 | : txn_id(txn_id_), tablet_id(tablet_id_) {} |
98 | 0 | auto operator<=>(const TxnKey&) const = default; |
99 | | }; |
100 | | |
101 | | struct TxnVal { |
102 | | RowsetSharedPtr rowset; |
103 | | int64_t txn_expiration; |
104 | | std::shared_ptr<PartialUpdateInfo> partial_update_info; |
105 | | std::shared_ptr<PublishStatus> publish_status = nullptr; |
106 | | // used to determine if the retry needs to re-calculate the delete bitmap |
107 | | TxnPublishInfo publish_info; |
108 | 0 | TxnVal() : txn_expiration(0) {}; |
109 | | TxnVal(RowsetSharedPtr rowset_, int64_t txn_expiration_, |
110 | | std::shared_ptr<PartialUpdateInfo> partial_update_info_, |
111 | | std::shared_ptr<PublishStatus> publish_status_) |
112 | 0 | : rowset(std::move(rowset_)), |
113 | 0 | txn_expiration(txn_expiration_), |
114 | 0 | partial_update_info(std::move(partial_update_info_)), |
115 | 0 | publish_status(std::move(publish_status_)) {} |
116 | | }; |
117 | | |
118 | | std::map<TxnKey, TxnVal> _txn_map; |
119 | | std::multimap<int64_t, TxnKey> _expiration_txn; |
120 | | // Lightweight markers for empty/skipped rowsets (only stores TxnKey, ~16 bytes per entry) |
121 | | // Used to track empty rowsets that were not committed to meta-service |
122 | | std::set<TxnKey> _empty_rowset_markers; |
123 | | std::shared_mutex _rwlock; |
124 | | std::shared_ptr<Thread> _clean_thread; |
125 | | CountDownLatch _stop_latch; |
126 | | }; |
127 | | |
128 | | } // namespace doris |