be/src/storage/changelog.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <limits> |
21 | | #include <map> |
22 | | #include <memory> |
23 | | #include <mutex> |
24 | | #include <string> |
25 | | #include <string_view> |
26 | | #include <utility> |
27 | | #include <vector> |
28 | | |
29 | | #include "common/logging.h" // DCHECK |
30 | | #include "common/status.h" |
31 | | #include "exec/sink/autoinc_buffer.h" |
32 | | #include "storage/changelog_types.h" |
33 | | #include "storage/olap_common.h" |
34 | | #include "storage/olap_define.h" // DataWriteType |
35 | | #include "storage/rowset/rowset_fwd.h" |
36 | | #include "storage/tablet/tablet_fwd.h" // BaseTabletSPtr |
37 | | #include "storage/tablet/tablet_schema.h" // TabletSchemaSPtr |
38 | | #include "storage/utils.h" // CHANGELOG_*_COL |
39 | | |
40 | | namespace doris { |
41 | | |
42 | | class AutoIncIDBuffer; |
43 | | struct PartialUpdateInfo; |
44 | | struct MowContext; |
45 | | |
46 | | namespace changelog { |
47 | | |
48 | | // ATTN(changelog-rename): formerly "__BEFORE__<col>__" |
49 | 366 | inline std::string build_before_column_name(std::string_view name) { |
50 | 366 | std::string before_name = "__DORIS_CHANGELOG_BEFORE_"; |
51 | 366 | before_name.append(name.data(), name.size()); |
52 | 366 | before_name.append("__"); |
53 | 366 | return before_name; |
54 | 366 | } |
55 | | |
56 | | } // namespace changelog |
57 | | |
58 | | constexpr int64_t kChangelogLsnAutoIncId = -1; |
59 | | |
60 | | // Allocate per-row LSNs. |
61 | | // The caller must provide a valid auto-inc buffer (typically from GlobalAutoIncBuffers). |
62 | | inline Status allocate_lsn(const std::shared_ptr<AutoIncIDBuffer>& lsn_buffer, size_t num_rows, |
63 | 7 | std::vector<int64_t>& lsn_ids) { |
64 | 7 | if (lsn_buffer == nullptr) { |
65 | 0 | return Status::InternalError("try to get lsn buffer but null"); |
66 | 0 | } |
67 | 7 | DCHECK(num_rows > 0); |
68 | | |
69 | 7 | std::vector<std::pair<int64_t, size_t>> ranges; |
70 | 7 | RETURN_IF_ERROR(lsn_buffer->sync_request_ids(num_rows, &ranges)); |
71 | | |
72 | 7 | lsn_ids.clear(); |
73 | 7 | lsn_ids.reserve(num_rows); |
74 | 7 | for (const auto& [start, length] : ranges) { |
75 | 18 | for (size_t i = 0; i < length; ++i) { |
76 | 11 | DCHECK_LE(start, std::numeric_limits<int64_t>::max() - static_cast<int64_t>(i)); |
77 | 11 | lsn_ids.push_back(start + static_cast<int64_t>(i)); |
78 | 11 | } |
79 | 7 | } |
80 | 7 | DCHECK_EQ(lsn_ids.size(), num_rows); |
81 | 7 | return Status::OK(); |
82 | 7 | } |
83 | | |
84 | | constexpr int64_t kTsoLogicalBits = 18; |
85 | | |
86 | 29 | inline int64_t extract_tso_physical_time(int64_t tso) { |
87 | 29 | return tso <= 0 ? 0 : tso >> kTsoLogicalBits; |
88 | 29 | } |
89 | | |
90 | 28 | inline int64_t changelog_ttl_cutoff_tso(int64_t reference_tso, int64_t ttl_seconds) { |
91 | 28 | DCHECK_GT(reference_tso, 0); |
92 | 28 | DCHECK_GE(ttl_seconds, 0); |
93 | 28 | const int64_t reference_ms = extract_tso_physical_time(reference_tso); |
94 | 28 | if (ttl_seconds > reference_ms / 1000) { |
95 | 2 | return 0; |
96 | 2 | } |
97 | 26 | constexpr int64_t kMaxLogical = (1L << kTsoLogicalBits) - 1; |
98 | 26 | return ((reference_ms - ttl_seconds * 1000) << kTsoLogicalBits) | kMaxLogical; |
99 | 28 | } |
100 | | |
101 | | namespace segment_v2 { |
102 | | |
103 | | class SegmentAllocatedLsnMap { |
104 | | public: |
105 | 67 | void insert_segment_allocated_lsns(int64_t seg_id, ConstAllocatedLsnVectorSharedPtr lsn_ids) { |
106 | 67 | std::lock_guard<std::mutex> l(_mutex); |
107 | 67 | _seg_id_to_lsn_ids.emplace(seg_id, std::move(lsn_ids)); |
108 | 67 | } |
109 | | |
110 | 55 | void remove_segment(int64_t seg_id) { |
111 | 55 | std::lock_guard<std::mutex> l(_mutex); |
112 | 55 | _seg_id_to_lsn_ids.erase(seg_id); |
113 | 55 | } |
114 | | |
115 | 6 | bool contains_segment(int64_t seg_id) const { |
116 | 6 | std::lock_guard<std::mutex> l(_mutex); |
117 | 6 | return _seg_id_to_lsn_ids.count(seg_id) > 0; |
118 | 6 | } |
119 | | |
120 | 54 | ConstAllocatedLsnVectorSharedPtr get_segment_allocated_lsns(int64_t seg_id) const { |
121 | 54 | std::lock_guard<std::mutex> l(_mutex); |
122 | 54 | auto it = _seg_id_to_lsn_ids.find(seg_id); |
123 | 54 | DCHECK(it != _seg_id_to_lsn_ids.end()) |
124 | 0 | << "SegmentAllocatedLsnMap::get_segment_allocated_lsns missing seg_id=" << seg_id |
125 | 0 | << ", existing_seg_ids=[" << ([&] { |
126 | 0 | std::string s; |
127 | 0 | for (const auto& [id, _] : _seg_id_to_lsn_ids) { |
128 | 0 | if (!s.empty()) { |
129 | 0 | s.push_back(','); |
130 | 0 | } |
131 | 0 | s.append(std::to_string(id)); |
132 | 0 | } |
133 | 0 | return s; |
134 | 0 | }()) |
135 | 0 | << "]"; |
136 | 54 | return it->second; |
137 | 54 | } |
138 | | |
139 | | private: |
140 | | mutable std::mutex _mutex; |
141 | | std::map<int64_t, ConstAllocatedLsnVectorSharedPtr> _seg_id_to_lsn_ids; |
142 | | }; |
143 | | |
144 | | struct SegmentWriteChangelogOptions { |
145 | | public: |
146 | | bool write_before = false; |
147 | | |
148 | | // source context, used for retrieving historical row and building changelog block |
149 | | struct SourceWriteDataOptions { |
150 | | BaseTabletSPtr base_tablet = nullptr; |
151 | | TabletSchemaSPtr tablet_schema = nullptr; |
152 | | std::shared_ptr<PartialUpdateInfo> partial_update_info; |
153 | | std::shared_ptr<MowContext> mow_context; |
154 | | bool is_transient_rowset_writer = false; |
155 | | DataWriteType source_write_type = DataWriteType::TYPE_DEFAULT; |
156 | | // input rowset's changelog rowset, read LSN from it at publish time |
157 | | RowsetSharedPtr changelog_rowset = nullptr; |
158 | | } source; |
159 | | }; |
160 | | } // namespace segment_v2 |
161 | | |
162 | | } // namespace doris |