be/src/cloud/cloud_committed_rs_mgr.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <map> |
21 | | #include <memory> |
22 | | #include <shared_mutex> |
23 | | |
24 | | #include "common/status.h" |
25 | | #include "storage/rowset/rowset_fwd.h" |
26 | | #include "util/countdown_latch.h" |
27 | | |
28 | | namespace doris { |
29 | | #include "common/compile_check_begin.h" |
30 | | class Thread; |
31 | | |
32 | | // Manages temporary rowset meta for cloud storage transactions in memory. |
33 | | // This cache stores rowset meta produced during import operations after they |
34 | | // are committed to MS. After the load txn was committed in MS finally, FE/BE will |
35 | | // notifies the final version/visible_ts, BE can update and promote these |
36 | | // temporary rowsets to the tablet meta without fetching from MS in later sync_rowsets(). |
37 | | class CloudCommittedRSMgr { |
38 | | public: |
39 | | CloudCommittedRSMgr(); |
40 | | ~CloudCommittedRSMgr(); |
41 | | |
42 | | Status init(); |
43 | | |
44 | | void add_committed_rowset(int64_t txn_id, int64_t tablet_id, RowsetMetaSharedPtr rowset_meta, |
45 | | int64_t expiration_time); |
46 | | |
47 | | Result<std::pair<RowsetMetaSharedPtr, int64_t>> get_committed_rowset(int64_t txn_id, |
48 | | int64_t tablet_id); |
49 | | |
50 | | void remove_committed_rowset(int64_t txn_id, int64_t tablet_id); |
51 | | |
52 | | void remove_expired_committed_rowsets(); |
53 | | |
54 | | void mark_empty_rowset(int64_t txn_id, int64_t tablet_id, int64_t txn_expiration); |
55 | | |
56 | | private: |
57 | | void _clean_thread_callback(); |
58 | | |
59 | | struct TxnTabletKey { |
60 | | int64_t txn_id; |
61 | | int64_t tablet_id; |
62 | | |
63 | | TxnTabletKey(int64_t txn_id_, int64_t tablet_id_) |
64 | 380k | : txn_id(txn_id_), tablet_id(tablet_id_) {} |
65 | | |
66 | 15.1M | auto operator<=>(const TxnTabletKey&) const = default; |
67 | | }; |
68 | | |
69 | | struct CommittedRowsetValue { |
70 | | RowsetMetaSharedPtr rowset_meta; |
71 | | int64_t expiration_time; // seconds since epoch |
72 | | |
73 | | CommittedRowsetValue(RowsetMetaSharedPtr rowset_meta_, int64_t expiration_time_) |
74 | 35.0k | : rowset_meta(std::move(rowset_meta_)), expiration_time(expiration_time_) {} |
75 | | }; |
76 | | |
77 | | // Map: <txn_id, tablet_id> -> <rowset_meta, expiration_time> |
78 | | std::map<TxnTabletKey, CommittedRowsetValue> _committed_rs_map; |
79 | | // Multimap for efficient expiration cleanup: expiration_time -> <txn_id, tablet_id> |
80 | | std::multimap<int64_t, TxnTabletKey> _expiration_map; |
81 | | std::map<TxnTabletKey, int64_t /* expiration_time */> _empty_rowset_markers; |
82 | | std::shared_mutex _rwlock; |
83 | | std::shared_ptr<Thread> _clean_thread; |
84 | | CountDownLatch _stop_latch; |
85 | | }; |
86 | | #include "common/compile_check_end.h" |
87 | | } // namespace doris |