be/src/storage/changelog.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <limits> |
21 | | #include <map> |
22 | | #include <memory> |
23 | | #include <mutex> |
24 | | #include <string> |
25 | | #include <string_view> |
26 | | #include <utility> |
27 | | #include <vector> |
28 | | |
29 | | #include "common/logging.h" // DCHECK |
30 | | #include "common/status.h" |
31 | | #include "exec/sink/autoinc_buffer.h" |
32 | | #include "storage/olap_common.h" |
33 | | #include "storage/olap_define.h" // DataWriteType |
34 | | #include "storage/rowset/rowset_fwd.h" |
35 | | #include "storage/tablet/tablet_fwd.h" // BaseTabletSPtr |
36 | | #include "storage/tablet/tablet_schema.h" // TabletSchemaSPtr |
37 | | #include "storage/utils.h" // CHANGELOG_*_COL |
38 | | |
39 | | namespace doris { |
40 | | |
41 | | class AutoIncIDBuffer; |
42 | | struct PartialUpdateInfo; |
43 | | struct MowContext; |
44 | | |
45 | | // Changelog op type. |
46 | | // NOTE: The value is persisted into changelog data, so keep it stable. |
47 | | static constexpr int64_t CHANGELOG_OP_APPEND = 0; |
48 | | static constexpr int64_t CHANGELOG_OP_UPDATE = 1; |
49 | | static constexpr int64_t CHANGELOG_OP_DELETE = 2; |
50 | | |
51 | | namespace changelog { |
52 | | |
53 | | // ATTN(changelog-rename): formerly "__BEFORE__<col>__" |
54 | 4.27k | inline std::string build_before_column_name(std::string_view name) { |
55 | 4.27k | std::string before_name = "__DORIS_CHANGELOG_BEFORE_"; |
56 | 4.27k | before_name.append(name.data(), name.size()); |
57 | 4.27k | before_name.append("__"); |
58 | 4.27k | return before_name; |
59 | 4.27k | } |
60 | | |
61 | | } // namespace changelog |
62 | | |
63 | | constexpr int64_t kChangelogLsnAutoIncId = -1; |
64 | | |
65 | | // Allocate per-row LSNs. |
66 | | // The caller must provide a valid auto-inc buffer (typically from GlobalAutoIncBuffers). |
67 | | inline Status allocate_lsn(const std::shared_ptr<AutoIncIDBuffer>& lsn_buffer, size_t num_rows, |
68 | 769 | std::vector<int64_t>& lsn_ids) { |
69 | 769 | if (lsn_buffer == nullptr) { |
70 | 0 | return Status::InternalError("try to get lsn buffer but null"); |
71 | 0 | } |
72 | 769 | DCHECK(num_rows > 0); |
73 | | |
74 | 769 | std::vector<std::pair<int64_t, size_t>> ranges; |
75 | 769 | RETURN_IF_ERROR(lsn_buffer->sync_request_ids(num_rows, &ranges)); |
76 | | |
77 | 769 | lsn_ids.clear(); |
78 | 769 | lsn_ids.reserve(num_rows); |
79 | 769 | for (const auto& [start, length] : ranges) { |
80 | 2.02k | for (size_t i = 0; i < length; ++i) { |
81 | 1.25k | DCHECK_LE(start, std::numeric_limits<int64_t>::max() - static_cast<int64_t>(i)); |
82 | 1.25k | lsn_ids.push_back(start + static_cast<int64_t>(i)); |
83 | 1.25k | } |
84 | 769 | } |
85 | 769 | DCHECK_EQ(lsn_ids.size(), num_rows); |
86 | 769 | return Status::OK(); |
87 | 769 | } |
88 | | |
89 | | constexpr int64_t kTsoLogicalBits = 18; |
90 | | |
91 | 329k | inline int64_t extract_tso_physical_time(int64_t tso) { |
92 | 329k | return tso <= 0 ? 0 : tso >> kTsoLogicalBits; |
93 | 329k | } |
94 | | |
95 | 329k | inline int64_t changelog_ttl_cutoff_tso(int64_t reference_tso, int64_t ttl_seconds) { |
96 | 329k | DCHECK_GT(reference_tso, 0); |
97 | 329k | DCHECK_GE(ttl_seconds, 0); |
98 | 329k | const int64_t reference_ms = extract_tso_physical_time(reference_tso); |
99 | 329k | if (ttl_seconds > reference_ms / 1000) { |
100 | 2 | return 0; |
101 | 2 | } |
102 | 329k | constexpr int64_t kMaxLogical = (1L << kTsoLogicalBits) - 1; |
103 | 329k | return ((reference_ms - ttl_seconds * 1000) << kTsoLogicalBits) | kMaxLogical; |
104 | 329k | } |
105 | | |
106 | | namespace segment_v2 { |
107 | | |
108 | | class SegmentAllocatedLsnMap { |
109 | | public: |
110 | 981 | void insert_segment_allocated_lsns(int64_t seg_id, ConstAllocatedLsnVectorSharedPtr lsn_ids) { |
111 | 981 | std::lock_guard<std::mutex> l(_mutex); |
112 | 981 | _seg_id_to_lsn_ids.emplace(seg_id, std::move(lsn_ids)); |
113 | 981 | } |
114 | | |
115 | 1.88k | void remove_segment(int64_t seg_id) { |
116 | 1.88k | std::lock_guard<std::mutex> l(_mutex); |
117 | 1.88k | _seg_id_to_lsn_ids.erase(seg_id); |
118 | 1.88k | } |
119 | | |
120 | | bool contains_segment(int64_t seg_id) const { |
121 | | std::lock_guard<std::mutex> l(_mutex); |
122 | | return _seg_id_to_lsn_ids.count(seg_id) > 0; |
123 | | } |
124 | | |
125 | 969 | ConstAllocatedLsnVectorSharedPtr get_segment_allocated_lsns(int64_t seg_id) const { |
126 | 969 | std::lock_guard<std::mutex> l(_mutex); |
127 | 969 | auto it = _seg_id_to_lsn_ids.find(seg_id); |
128 | 18.4E | DCHECK(it != _seg_id_to_lsn_ids.end()) |
129 | 18.4E | << "SegmentAllocatedLsnMap::get_segment_allocated_lsns missing seg_id=" << seg_id |
130 | 18.4E | << ", existing_seg_ids=[" << ([&] { |
131 | 0 | std::string s; |
132 | 0 | for (const auto& [id, _] : _seg_id_to_lsn_ids) { |
133 | 0 | if (!s.empty()) { |
134 | 0 | s.push_back(','); |
135 | 0 | } |
136 | 0 | s.append(std::to_string(id)); |
137 | 0 | } |
138 | 0 | return s; |
139 | 0 | }()) |
140 | 18.4E | << "]"; |
141 | 969 | return it->second; |
142 | 969 | } |
143 | | |
144 | | private: |
145 | | mutable std::mutex _mutex; |
146 | | std::map<int64_t, ConstAllocatedLsnVectorSharedPtr> _seg_id_to_lsn_ids; |
147 | | }; |
148 | | |
149 | | struct SegmentWriteChangelogOptions { |
150 | | public: |
151 | | bool write_before = false; |
152 | | |
153 | | // source context, used for retrieving historical row and building changelog block |
154 | | struct SourceWriteDataOptions { |
155 | | BaseTabletSPtr base_tablet = nullptr; |
156 | | TabletSchemaSPtr tablet_schema = nullptr; |
157 | | std::shared_ptr<PartialUpdateInfo> partial_update_info; |
158 | | std::shared_ptr<MowContext> mow_context; |
159 | | bool is_transient_rowset_writer = false; |
160 | | DataWriteType source_write_type = DataWriteType::TYPE_DEFAULT; |
161 | | // input rowset's changelog rowset, read LSN from it at publish time |
162 | | RowsetSharedPtr changelog_rowset = nullptr; |
163 | | } source; |
164 | | }; |
165 | | } // namespace segment_v2 |
166 | | |
167 | | } // namespace doris |