be/src/storage/rowset_builder.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "storage/rowset_builder.h" |
19 | | |
20 | | #include <brpc/controller.h> |
21 | | #include <fmt/format.h> |
22 | | |
23 | | #include <filesystem> |
24 | | #include <memory> |
25 | | #include <ostream> |
26 | | #include <string> |
27 | | #include <utility> |
28 | | |
29 | | // IWYU pragma: no_include <opentelemetry/common/threadlocal.h> |
30 | | #include "common/compiler_util.h" // IWYU pragma: keep |
31 | | #include "common/config.h" |
32 | | #include "common/status.h" |
33 | | #include "core/block/block.h" |
34 | | #include "io/fs/file_system.h" |
35 | | #include "io/fs/file_writer.h" // IWYU pragma: keep |
36 | | #include "runtime/memory/global_memory_arbitrator.h" |
37 | | #include "storage/delete/calc_delete_bitmap_executor.h" |
38 | | #include "storage/olap_define.h" |
39 | | #include "storage/partial_update_info.h" |
40 | | #include "storage/rowset/beta_rowset.h" |
41 | | #include "storage/rowset/beta_rowset_writer.h" |
42 | | #include "storage/rowset/group_rowset_writer.h" |
43 | | #include "storage/rowset/pending_rowset_helper.h" |
44 | | #include "storage/rowset/rowset_factory.h" |
45 | | #include "storage/rowset/rowset_meta.h" |
46 | | #include "storage/rowset/rowset_meta_manager.h" |
47 | | #include "storage/rowset/rowset_writer.h" |
48 | | #include "storage/rowset/rowset_writer_context.h" |
49 | | #include "storage/schema_change/schema_change.h" |
50 | | #include "storage/storage_engine.h" |
51 | | #include "storage/tablet/tablet_manager.h" |
52 | | #include "storage/tablet/tablet_meta.h" |
53 | | #include "storage/tablet/tablet_schema.h" |
54 | | #include "storage/tablet_info.h" |
55 | | #include "storage/txn/txn_manager.h" |
56 | | #include "util/brpc_client_cache.h" |
57 | | #include "util/brpc_closure.h" |
58 | | #include "util/debug_points.h" |
59 | | #include "util/mem_info.h" |
60 | | #include "util/stopwatch.hpp" |
61 | | #include "util/time.h" |
62 | | #include "util/trace.h" |
63 | | |
64 | | namespace doris { |
65 | | using namespace ErrorCode; |
66 | | |
67 | | BaseRowsetBuilder::BaseRowsetBuilder(const WriteRequest& req, RuntimeProfile* profile) |
68 | 279k | : _req(req), _tablet_schema(std::make_shared<TabletSchema>()) { |
69 | 279k | if (profile != nullptr) { |
70 | 13.7k | _init_profile(profile); |
71 | 13.7k | } |
72 | 279k | } |
73 | | |
74 | | RowsetBuilder::RowsetBuilder(StorageEngine& engine, const WriteRequest& req, |
75 | | RuntimeProfile* profile) |
76 | 17.2k | : BaseRowsetBuilder(req, profile), _engine(engine) {} |
77 | | |
78 | | RowBinlogRowsetBuilder::RowBinlogRowsetBuilder(StorageEngine& engine, const WriteRequest& req, |
79 | | RuntimeProfile* profile) |
80 | 1 | : RowsetBuilder(engine, req, profile) {} |
81 | | |
82 | 13.7k | void BaseRowsetBuilder::_init_profile(RuntimeProfile* profile) { |
83 | 13.7k | DCHECK(profile != nullptr); |
84 | 13.7k | if (_req.write_req_type == WriteRequestType::GROUP) { |
85 | 1 | _profile = profile->create_child(fmt::format("GroupRowsetBuilder {}", _req.tablet_id), true, |
86 | 1 | true); |
87 | 1 | return; |
88 | 1 | } |
89 | | |
90 | 13.7k | _profile = profile->create_child( |
91 | 13.7k | fmt::format( |
92 | 13.7k | "RowsetBuilder {} {}", _req.tablet_id, |
93 | 13.7k | _req.write_req_type == WriteRequestType::ROW_BINLOG ? "binlog<row>" : "data"), |
94 | 13.7k | true, true); |
95 | 13.7k | _build_rowset_timer = ADD_TIMER(_profile, "BuildRowsetTime"); |
96 | 13.7k | _submit_delete_bitmap_timer = ADD_TIMER(_profile, "DeleteBitmapSubmitTime"); |
97 | 13.7k | _wait_delete_bitmap_timer = ADD_TIMER(_profile, "DeleteBitmapWaitTime"); |
98 | 13.7k | } |
99 | | |
100 | 0 | void RowsetBuilder::_init_profile(RuntimeProfile* profile) { |
101 | 0 | DCHECK(profile != nullptr); |
102 | 0 | BaseRowsetBuilder::_init_profile(profile); |
103 | 0 | _commit_txn_timer = ADD_TIMER(_profile, "CommitTxnTime"); |
104 | 0 | } |
105 | | |
106 | 279k | BaseRowsetBuilder::~BaseRowsetBuilder() { |
107 | 279k | if (!_is_init) { |
108 | 91.5k | return; |
109 | 91.5k | } |
110 | | |
111 | 187k | if (_calc_delete_bitmap_token != nullptr) { |
112 | 187k | _calc_delete_bitmap_token->cancel(); |
113 | 187k | } |
114 | 187k | } |
115 | | |
116 | 17.2k | RowsetBuilder::~RowsetBuilder() { |
117 | 17.2k | if (_is_init && !_is_committed) { |
118 | | // For txn rowset builders, we need to rollback txn when necessary. |
119 | 7 | _garbage_collection(is_data_builder()); |
120 | 7 | } |
121 | 17.2k | } |
122 | | |
123 | 129k | Tablet* RowsetBuilder::tablet() { |
124 | 129k | return static_cast<Tablet*>(_tablet.get()); |
125 | 129k | } |
126 | | |
127 | 7 | void RowsetBuilder::_garbage_collection(bool cancel_txn) { |
128 | 7 | Status rollback_status; |
129 | 7 | bool need_clean = true; |
130 | 7 | if (tablet() != nullptr && cancel_txn) { |
131 | 6 | TxnManager* txn_mgr = _engine.txn_manager(); |
132 | 6 | rollback_status = txn_mgr->rollback_txn(_req.partition_id, *tablet(), _req.txn_id); |
133 | 6 | need_clean = rollback_status.ok(); |
134 | 6 | } |
135 | | // has to check rollback status, because the rowset maybe committed in this thread and |
136 | | // published in another thread, then rollback will fail. |
137 | | // when rollback failed should not delete rowset |
138 | 7 | if (need_clean) { |
139 | 7 | _engine.add_unused_rowset(_rowset); |
140 | 7 | for (auto& rs : _attach_rowsets) { |
141 | 0 | _engine.add_unused_rowset(rs); |
142 | 0 | } |
143 | 7 | } |
144 | 7 | } |
145 | | |
146 | 57.7k | Status BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_context) { |
147 | 57.7k | DCHECK(is_data_builder()); |
148 | | |
149 | 57.7k | std::lock_guard<std::shared_mutex> lck(tablet()->get_header_lock()); |
150 | 57.7k | _max_version_in_flush_phase = tablet()->max_version_unlocked(); |
151 | 57.7k | std::vector<RowsetSharedPtr> rowset_ptrs; |
152 | | // tablet is under alter process. The delete bitmap will be calculated after conversion. |
153 | 57.7k | if (tablet()->tablet_state() == TABLET_NOTREADY) { |
154 | | // Disable 'partial_update' when the tablet is undergoing a 'schema changing process' |
155 | 30 | if (_req.table_schema_param->is_partial_update()) { |
156 | 0 | return Status::InternalError( |
157 | 0 | "Unable to do 'partial_update' when " |
158 | 0 | "the tablet is undergoing a 'schema changing process'"); |
159 | 0 | } |
160 | 30 | _rowset_ids->clear(); |
161 | 57.6k | } else { |
162 | 57.6k | RETURN_IF_ERROR( |
163 | 57.6k | tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase, _rowset_ids.get())); |
164 | 57.6k | rowset_ptrs = tablet()->get_rowset_by_ids(_rowset_ids.get()); |
165 | 57.6k | } |
166 | 57.7k | _delete_bitmap = std::make_shared<DeleteBitmap>(tablet()->tablet_id()); |
167 | 57.7k | mow_context = std::make_shared<MowContext>(_max_version_in_flush_phase, _req.txn_id, |
168 | 57.7k | _rowset_ids, rowset_ptrs, _delete_bitmap); |
169 | 57.7k | return Status::OK(); |
170 | 57.7k | } |
171 | | |
172 | 16.2k | Status RowsetBuilder::check_tablet_version_count() { |
173 | 16.2k | DCHECK(is_data_builder()); |
174 | | |
175 | 16.2k | auto max_version_config = _tablet->max_version_config(); |
176 | 16.2k | auto version_count = tablet()->version_count(); |
177 | 16.2k | DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version", |
178 | 16.2k | { version_count = INT_MAX; }); |
179 | | // Trigger TOO MANY VERSION error first |
180 | 16.2k | if (version_count > max_version_config) { |
181 | 0 | return Status::Error<TOO_MANY_VERSION>( |
182 | 0 | "failed to init rowset builder. version count: {}, exceed limit: {}, " |
183 | 0 | "tablet: {}. Please reduce the frequency of loading data or adjust the " |
184 | 0 | "max_tablet_version_num or time_series_max_tablet_version_num in be.conf to a " |
185 | 0 | "larger value.", |
186 | 0 | version_count, max_version_config, _tablet->tablet_id()); |
187 | 0 | } |
188 | | // (TODO Refrain) Maybe we can use a configurable param instead of hardcoded values '100'. |
189 | | // max_version_config must > 100, otherwise silent errors will occur. |
190 | 16.2k | if ((!config::disable_auto_compaction && |
191 | 16.2k | !_tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) && |
192 | 16.2k | (version_count > max_version_config - 100) && |
193 | 16.2k | !GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { |
194 | | // Trigger compaction |
195 | 0 | auto st = _engine.submit_compaction_task(std::static_pointer_cast<Tablet>(tablet_sptr()), |
196 | 0 | CompactionType::CUMULATIVE_COMPACTION, true, true, |
197 | 0 | 2); |
198 | 0 | if (!st.ok()) [[unlikely]] { |
199 | 0 | LOG(WARNING) << "failed to trigger compaction, tablet_id=" << _tablet->tablet_id() |
200 | 0 | << " : " << st; |
201 | 0 | } |
202 | 0 | } |
203 | 16.2k | return Status::OK(); |
204 | 16.2k | } |
205 | | |
206 | 16.2k | Status RowsetBuilder::prepare_txn() { |
207 | 16.2k | DCHECK(is_data_builder()); |
208 | 16.2k | return tablet()->prepare_txn(_req.partition_id, _req.txn_id, _req.load_id, false); |
209 | 16.2k | } |
210 | | |
211 | 16.2k | Status RowsetBuilder::init() { |
212 | 16.2k | RowsetWriterContext context; |
213 | | |
214 | 16.2k | RETURN_IF_ERROR(_init_context_common_fields(context)); |
215 | | |
216 | 16.2k | std::shared_ptr<MowContext> mow_context; |
217 | 16.2k | if (_tablet->enable_unique_key_merge_on_write()) { |
218 | 6.05k | RETURN_IF_ERROR(init_mow_context(mow_context)); |
219 | 6.05k | } |
220 | | |
221 | 16.2k | RETURN_IF_ERROR(check_tablet_version_count()); |
222 | | |
223 | 16.2k | auto version_count = tablet()->version_count() + tablet()->stale_version_count(); |
224 | 16.2k | if (tablet()->avg_rs_meta_serialize_size() * version_count > |
225 | 16.2k | config::tablet_meta_serialize_size_limit) { |
226 | 0 | return Status::Error<TOO_MANY_VERSION>( |
227 | 0 | "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, " |
228 | 0 | "tablet: {}. Please reduce the frequency of loading data or adjust the " |
229 | 0 | "max_tablet_version_num in be.conf to a larger value.", |
230 | 0 | tablet()->avg_rs_meta_serialize_size() * version_count, |
231 | 0 | config::tablet_meta_serialize_size_limit, _tablet->tablet_id()); |
232 | 0 | } |
233 | | |
234 | 16.2k | RETURN_IF_ERROR(prepare_txn()); |
235 | | |
236 | 16.2k | DBUG_EXECUTE_IF("BaseRowsetBuilder::init.check_partial_update_column_num", { |
237 | 16.2k | if (_req.table_schema_param->partial_update_input_columns().size() != |
238 | 16.2k | dp->param<int>("column_num")) { |
239 | 16.2k | return Status::InternalError("partial update input column num wrong!"); |
240 | 16.2k | }; |
241 | 16.2k | }) |
242 | | // build tablet schema in request level |
243 | 16.2k | RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(), |
244 | 16.2k | *_tablet->tablet_schema())); |
245 | | |
246 | 16.2k | context.mow_context = mow_context; |
247 | | |
248 | 16.2k | context.partial_update_info = _partial_update_info; |
249 | 16.2k | _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false)); |
250 | 16.2k | _rowset_id = context.rowset_id; |
251 | 16.2k | std::vector<RowsetId> tmp_pending_rowset_ids = {_rowset_id}; |
252 | 16.2k | tmp_pending_rowset_ids.resize(1 + _attach_rowset_ids.size()); |
253 | 16.2k | std::copy(_attach_rowset_ids.begin(), _attach_rowset_ids.end(), |
254 | 16.2k | tmp_pending_rowset_ids.begin() + 1); |
255 | 16.2k | _pending_rs_guard = _engine.pending_local_rowsets().add(tmp_pending_rowset_ids); |
256 | | |
257 | 16.2k | _calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor()->create_token(); |
258 | | |
259 | 16.2k | _is_init = true; |
260 | 16.2k | return Status::OK(); |
261 | 16.2k | } |
262 | | |
263 | 16.2k | Status BaseRowsetBuilder::_init_context_common_fields(RowsetWriterContext& context) { |
264 | 16.2k | _tablet = DORIS_TRY(ExecEnv::get_tablet(_req.tablet_id)); |
265 | | |
266 | 16.2k | context.txn_id = _req.txn_id; |
267 | 16.2k | context.load_id = _req.load_id; |
268 | 16.2k | context.db_id = _req.table_schema_param->db_id(); |
269 | 16.2k | context.table_id = _req.table_schema_param->table_id(); |
270 | 16.2k | context.rowset_state = PREPARED; |
271 | 16.2k | context.segments_overlap = OVERLAPPING; |
272 | 16.2k | context.tablet_schema = _tablet_schema; |
273 | 16.2k | context.newest_write_timestamp = UnixSeconds(); |
274 | 16.2k | context.tablet_id = _req.tablet_id; |
275 | 16.2k | context.index_id = _req.index_id; |
276 | 16.2k | context.tablet = _tablet; |
277 | 16.2k | context.enable_segcompaction = true; |
278 | 16.2k | if (_req.write_req_type == WriteRequestType::ROW_BINLOG || !_attach_rowset_ids.empty()) { |
279 | 2 | context.enable_segcompaction = false; |
280 | 2 | } |
281 | 16.2k | context.write_type = DataWriteType::TYPE_DIRECT; |
282 | 16.2k | context.write_file_cache = _req.write_file_cache; |
283 | | |
284 | 16.2k | return Status::OK(); |
285 | 16.2k | } |
286 | | |
287 | 187k | Status BaseRowsetBuilder::build_rowset() { |
288 | 187k | std::lock_guard<std::mutex> l(_lock); |
289 | 18.4E | DCHECK(_is_init) << "rowset builder is supposed be to initialized before " |
290 | 18.4E | "build_rowset() being called"; |
291 | | |
292 | 187k | SCOPED_TIMER(_build_rowset_timer); |
293 | | // use rowset meta manager to save meta |
294 | 187k | RETURN_NOT_OK_STATUS_WITH_WARN(_rowset_writer->build(_rowset), "fail to build rowset"); |
295 | 187k | return Status::OK(); |
296 | 187k | } |
297 | | |
298 | 1 | Status GroupRowsetBuilder::build_rowset() { |
299 | | // build binlog rowset first, then data rowset |
300 | | // If data rowset build fails after binlog has built some segments, we rely on the |
301 | | // RowsetBuilder/StorageEngine garbage-collection paths to clean up built-but-uncommitted files. |
302 | 1 | RETURN_IF_ERROR(_row_binlog_rowset_builder->build_rowset()); |
303 | 1 | return _txn_rs_builder->build_rowset(); |
304 | 1 | } |
305 | | |
306 | 187k | Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() { |
307 | 187k | DCHECK(is_data_builder()); |
308 | 187k | if (!_tablet->enable_unique_key_merge_on_write() || _rowset->num_segments() == 0) { |
309 | 164k | return Status::OK(); |
310 | 164k | } |
311 | 22.9k | std::lock_guard<std::mutex> l(_lock); |
312 | 22.9k | SCOPED_TIMER(_submit_delete_bitmap_timer); |
313 | 22.9k | if (_partial_update_info && _partial_update_info->is_flexible_partial_update()) { |
314 | 264 | if (_rowset->num_segments() > 1) { |
315 | | // in flexible partial update, when there are more one segment in one load, |
316 | | // we need to do alignment process for same keys between segments, we haven't |
317 | | // implemented it yet and just report an error when encouter this situation |
318 | 0 | return Status::NotSupported( |
319 | 0 | "too large input data in flexible partial update, Please " |
320 | 0 | "reduce the amount of data imported in a single load."); |
321 | 0 | } |
322 | 264 | } |
323 | | |
324 | 22.9k | auto* beta_rowset = reinterpret_cast<BetaRowset*>(_rowset.get()); |
325 | 22.9k | std::vector<segment_v2::SegmentSharedPtr> segments; |
326 | 22.9k | RETURN_IF_ERROR(beta_rowset->load_segments(&segments)); |
327 | 22.9k | if (segments.size() > 1) { |
328 | | // calculate delete bitmap between segments |
329 | 0 | if (config::enable_calc_delete_bitmap_between_segments_concurrently) { |
330 | 0 | RETURN_IF_ERROR(_calc_delete_bitmap_token->submit( |
331 | 0 | _tablet, _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap)); |
332 | 0 | } else { |
333 | 0 | RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments( |
334 | 0 | _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap)); |
335 | 0 | } |
336 | 0 | } |
337 | | |
338 | | // For partial update, we need to fill in the entire row of data, during the calculation |
339 | | // of the delete bitmap. This operation is resource-intensive, and we need to minimize |
340 | | // the number of times it occurs. Therefore, we skip this operation here. |
341 | 22.9k | if (_partial_update_info->is_partial_update()) { |
342 | | // for partial update, the delete bitmap calculation is done while append_block() |
343 | | // we print it's summarize logs here before commit. |
344 | 1.61k | LOG(INFO) << fmt::format( |
345 | 1.61k | "{} calc delete bitmap summary before commit: tablet({}), txn_id({}), " |
346 | 1.61k | "rowset_ids({}), cur max_version({}), bitmap num({}), bitmap_cardinality({}), num " |
347 | 1.61k | "rows updated({}), num rows new added({}), num rows deleted({}), total rows({})", |
348 | 1.61k | _partial_update_info->partial_update_mode_str(), tablet()->tablet_id(), _req.txn_id, |
349 | 1.61k | _rowset_ids->size(), rowset_writer()->context().mow_context->max_version, |
350 | 1.61k | _delete_bitmap->get_delete_bitmap_count(), _delete_bitmap->cardinality(), |
351 | 1.61k | rowset_writer()->num_rows_updated(), rowset_writer()->num_rows_new_added(), |
352 | 1.61k | rowset_writer()->num_rows_deleted(), rowset_writer()->num_rows()); |
353 | 1.61k | return Status::OK(); |
354 | 1.61k | } |
355 | | |
356 | 22.9k | LOG(INFO) << "submit calc delete bitmap task to executor, tablet_id: " << tablet()->tablet_id() |
357 | 21.3k | << ", txn_id: " << _req.txn_id; |
358 | 21.3k | return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset, *_rowset_ids, |
359 | 21.3k | _delete_bitmap, segments, _req.txn_id, |
360 | 21.3k | _calc_delete_bitmap_token.get(), nullptr); |
361 | 22.9k | } |
362 | | |
363 | 187k | Status BaseRowsetBuilder::wait_calc_delete_bitmap() { |
364 | 187k | DCHECK(is_data_builder()); |
365 | 187k | if (!_tablet->enable_unique_key_merge_on_write() || _partial_update_info->is_partial_update()) { |
366 | 133k | return Status::OK(); |
367 | 133k | } |
368 | 54.1k | std::lock_guard<std::mutex> l(_lock); |
369 | 54.1k | SCOPED_TIMER(_wait_delete_bitmap_timer); |
370 | 54.1k | RETURN_IF_ERROR(_calc_delete_bitmap_token->wait()); |
371 | 54.1k | return Status::OK(); |
372 | 54.1k | } |
373 | | |
374 | 16.2k | Status RowsetBuilder::commit_txn() { |
375 | 16.2k | DCHECK(is_data_builder()); |
376 | 16.2k | if (tablet()->enable_unique_key_merge_on_write() && |
377 | 16.2k | config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0 && |
378 | 16.2k | tablet()->tablet_state() != TABLET_NOTREADY) { |
379 | 2.05k | auto st = tablet()->check_delete_bitmap_correctness( |
380 | 2.05k | _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids); |
381 | 2.05k | if (!st.ok()) { |
382 | 0 | LOG(WARNING) << fmt::format( |
383 | 0 | "[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] " |
384 | 0 | "delete bitmap correctness check failed in commit phase!", |
385 | 0 | _req.tablet_id, _req.txn_id, UniqueId(_req.load_id).to_string(), |
386 | 0 | _req.partition_id); |
387 | 0 | return st; |
388 | 0 | } |
389 | 2.05k | } |
390 | 16.2k | std::lock_guard<std::mutex> l(_lock); |
391 | 16.2k | SCOPED_TIMER(_commit_txn_timer); |
392 | | |
393 | | // Transfer ownership of `PendingRowsetGuard` to `TxnManager` |
394 | 16.2k | Status res = _engine.txn_manager()->commit_txn( |
395 | 16.2k | _req.partition_id, *tablet(), _req.txn_id, _req.load_id, _rowset, |
396 | 16.2k | std::move(_pending_rs_guard), false, _partial_update_info, &_attach_rowsets); |
397 | | |
398 | 16.2k | if (!res && !res.is<PUSH_TRANSACTION_ALREADY_EXIST>()) { |
399 | 0 | LOG(WARNING) << "Failed to commit txn: " << _req.txn_id |
400 | 0 | << " for rowset: " << _rowset->rowset_id(); |
401 | 0 | return res; |
402 | 0 | } |
403 | 16.2k | if (_tablet->enable_unique_key_merge_on_write()) { |
404 | | // no need to update binlog_delvec, it'll be updated in publish phase |
405 | 6.05k | _engine.txn_manager()->set_txn_related_delete_bitmap( |
406 | 6.05k | _req.partition_id, _req.txn_id, tablet()->tablet_id(), tablet()->tablet_uid(), true, |
407 | 6.05k | _delete_bitmap, *_rowset_ids, _partial_update_info); |
408 | 6.05k | } |
409 | | |
410 | 16.2k | _is_committed = true; |
411 | 16.2k | return Status::OK(); |
412 | 16.2k | } |
413 | | |
414 | 0 | Status BaseRowsetBuilder::cancel() { |
415 | 0 | std::lock_guard<std::mutex> l(_lock); |
416 | 0 | if (_is_cancelled) { |
417 | 0 | return Status::OK(); |
418 | 0 | } |
419 | 0 | if (_calc_delete_bitmap_token != nullptr) { |
420 | 0 | _calc_delete_bitmap_token->cancel(); |
421 | 0 | } |
422 | 0 | _is_cancelled = true; |
423 | 0 | return Status::OK(); |
424 | 0 | } |
425 | | |
426 | | Status BaseRowsetBuilder::_build_current_tablet_schema( |
427 | | int64_t index_id, const OlapTableSchemaParam* table_schema_param, |
428 | 187k | const TabletSchema& ori_tablet_schema) { |
429 | 187k | const OlapTableIndexSchema* index_schema = nullptr; |
430 | 187k | if (_req.write_req_type == WriteRequestType::ROW_BINLOG) { |
431 | 1 | const auto* row_binlog_index_schema = table_schema_param->row_binlog_index_schema(); |
432 | 1 | DCHECK(row_binlog_index_schema != nullptr); |
433 | 1 | DCHECK_EQ(row_binlog_index_schema->index_id, index_id); |
434 | 1 | index_schema = row_binlog_index_schema; |
435 | 187k | } else { |
436 | 203k | for (const auto* schema : table_schema_param->indexes()) { |
437 | 203k | if (schema->index_id == index_id) { |
438 | 186k | index_schema = schema; |
439 | 186k | break; |
440 | 186k | } |
441 | 203k | } |
442 | 187k | } |
443 | | |
444 | 187k | if (index_schema != nullptr && !index_schema->columns.empty() && |
445 | 187k | index_schema->columns[0]->unique_id() >= 0) { |
446 | 186k | _tablet_schema->shawdow_copy_without_columns(ori_tablet_schema); |
447 | 186k | _tablet_schema->build_current_tablet_schema( |
448 | 186k | index_id, cast_set<int32_t>(table_schema_param->version()), index_schema, |
449 | 186k | ori_tablet_schema); |
450 | 186k | } else { |
451 | 569 | _tablet_schema->copy_from(ori_tablet_schema); |
452 | 569 | } |
453 | 187k | if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version()) { |
454 | | // After schema change, should include extracted column |
455 | | // For example: a table has two columns, k and v |
456 | | // After adding a column v2, the schema version increases, max_version_schema needs to be updated. |
457 | | // _tablet_schema includes k, v, and v2 |
458 | | // if v is a variant, need to add the columns decomposed from the v to the _tablet_schema. |
459 | 1.29k | if (is_data_builder()) { |
460 | 1.29k | if (_tablet_schema->num_variant_columns() > 0) { |
461 | 105 | TabletSchemaSPtr max_version_schema = std::make_shared<TabletSchema>(); |
462 | 105 | max_version_schema->copy_from(*_tablet_schema); |
463 | 105 | max_version_schema->copy_extracted_columns(ori_tablet_schema); |
464 | 105 | _tablet->update_max_version_schema(max_version_schema); |
465 | 1.19k | } else { |
466 | 1.19k | _tablet->update_max_version_schema(_tablet_schema); |
467 | 1.19k | } |
468 | 1.29k | } |
469 | 1.29k | } |
470 | | |
471 | 187k | _tablet_schema->set_table_id(table_schema_param->table_id()); |
472 | 187k | _tablet_schema->set_db_id(table_schema_param->db_id()); |
473 | 187k | if (table_schema_param->is_partial_update()) { |
474 | 3.78k | _tablet_schema->set_auto_increment_column(table_schema_param->auto_increment_coulumn()); |
475 | 3.78k | } |
476 | | // set partial update columns info |
477 | 187k | if (is_data_builder()) { |
478 | 187k | _partial_update_info = std::make_shared<PartialUpdateInfo>(); |
479 | 187k | RETURN_IF_ERROR(_partial_update_info->init( |
480 | 187k | tablet()->tablet_id(), _req.txn_id, *_tablet_schema, |
481 | 187k | table_schema_param->unique_key_update_mode(), |
482 | 187k | table_schema_param->partial_update_new_key_policy(), |
483 | 187k | table_schema_param->partial_update_input_columns(), |
484 | 187k | table_schema_param->is_strict_mode(), table_schema_param->timestamp_ms(), |
485 | 187k | table_schema_param->nano_seconds(), table_schema_param->timezone(), |
486 | 187k | table_schema_param->auto_increment_coulumn(), |
487 | 187k | table_schema_param->sequence_map_col_uid(), _max_version_in_flush_phase)); |
488 | 187k | } |
489 | 187k | return Status::OK(); |
490 | 187k | } |
491 | | |
492 | | GroupRowsetBuilder::GroupRowsetBuilder(StorageEngine& engine, const WriteRequest& group_build_req, |
493 | | const WriteRequest& sub_data_req, |
494 | | const WriteRequest& sub_row_binlog_req, |
495 | | RuntimeProfile* profile) |
496 | 1 | : BaseRowsetBuilder(group_build_req, profile) { |
497 | 1 | DCHECK(group_build_req.write_req_type == WriteRequestType::GROUP && |
498 | 1 | sub_data_req.write_req_type == WriteRequestType::DATA && |
499 | 1 | sub_row_binlog_req.write_req_type == WriteRequestType::ROW_BINLOG); |
500 | 1 | _row_binlog_rowset_builder = |
501 | 1 | std::make_shared<RowBinlogRowsetBuilder>(engine, sub_row_binlog_req, profile); |
502 | 1 | _txn_rs_builder = std::make_shared<RowsetBuilder>(engine, sub_data_req, profile); |
503 | 1 | } |
504 | | |
505 | 1 | Status GroupRowsetBuilder::init() { |
506 | | // init binlog builder first so that its rowset id can be added into |
507 | | // PendingLocalRowsets before txn builder init. |
508 | 1 | RETURN_IF_ERROR(_row_binlog_rowset_builder->init()); |
509 | | // before init txn, need to add all rowset_ids into PendingLocalRowsets. |
510 | | // see https://github.com/apache/doris/pull/25921 |
511 | 1 | RETURN_IF_ERROR(_txn_rs_builder->attach_pending_rs_guard_to_txn( |
512 | 1 | _row_binlog_rowset_builder->rowset_id())); |
513 | 1 | RETURN_IF_ERROR(_txn_rs_builder->init()); |
514 | | |
515 | | // Create a GroupRowsetWriter that forwards flush to both underlying |
516 | | // RowsetWriters. |
517 | 1 | std::unique_ptr<doris::GroupRowsetWriter> group_writer; |
518 | 1 | RETURN_IF_ERROR(RowsetFactory::create_empty_group_rowset_writer(&group_writer)); |
519 | 1 | group_writer->set_data_writer(_txn_rs_builder->rowset_writer()); |
520 | 1 | group_writer->set_row_binlog_writer(_row_binlog_rowset_builder->rowset_writer()); |
521 | | |
522 | 1 | { |
523 | 1 | const auto& data_ctx = _txn_rs_builder->rowset_writer()->context(); |
524 | 1 | auto& binlog_ctx = const_cast<RowsetWriterContext&>( |
525 | 1 | _row_binlog_rowset_builder->rowset_writer()->context()); |
526 | 1 | auto& cfg = binlog_ctx.write_binlog_opt().write_binlog_config(); |
527 | 1 | cfg.source.tablet_schema = data_ctx.tablet_schema; |
528 | 1 | cfg.source.partial_update_info = data_ctx.partial_update_info; |
529 | 1 | cfg.source.mow_context = data_ctx.mow_context; |
530 | 1 | cfg.source.is_transient_rowset_writer = data_ctx.is_transient_rowset_writer; |
531 | 1 | cfg.source.source_write_type = data_ctx.write_type; |
532 | 1 | } |
533 | | |
534 | 1 | _rowset_writer = std::move(group_writer); |
535 | 1 | _is_init = true; |
536 | 1 | return Status::OK(); |
537 | 1 | } |
538 | | |
539 | 0 | Status GroupRowsetBuilder::submit_calc_delete_bitmap_task() { |
540 | 0 | return _txn_rs_builder->submit_calc_delete_bitmap_task(); |
541 | 0 | } |
542 | | |
543 | 0 | Status GroupRowsetBuilder::wait_calc_delete_bitmap() { |
544 | 0 | return _txn_rs_builder->wait_calc_delete_bitmap(); |
545 | 0 | } |
546 | | |
547 | 0 | Status GroupRowsetBuilder::commit_txn() { |
548 | | // Attach binlog rowset to txn rowset, so that commit/rollback and |
549 | | // clean-up are all handled by txn rowset builder. |
550 | 0 | RETURN_IF_ERROR(_txn_rs_builder->attach_rowset_to_txn(_row_binlog_rowset_builder->rowset())); |
551 | 0 | auto st = _txn_rs_builder->commit_txn(); |
552 | 0 | if (st.ok()) { |
553 | | // Avoid RowBinlogRowsetBuilder being cleaned in its base dtor. |
554 | 0 | RETURN_IF_ERROR(_row_binlog_rowset_builder->commit_txn()); |
555 | 0 | } |
556 | 0 | return st; |
557 | 0 | } |
558 | | |
559 | 1 | Status RowBinlogRowsetBuilder::init() { |
560 | 1 | RowsetWriterContext context; |
561 | | |
562 | 1 | RETURN_IF_ERROR(_init_context_common_fields(context)); |
563 | | |
564 | | // build tablet schema in request level |
565 | 1 | RETURN_IF_ERROR(_build_current_tablet_schema( |
566 | 1 | _req.index_id, _req.table_schema_param.get(), |
567 | 1 | *std::dynamic_pointer_cast<Tablet>(_tablet)->row_binlog_tablet_schema())); |
568 | 1 | context.write_binlog_opt().enable = true; |
569 | | |
570 | 1 | _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false)); |
571 | | // need to attach PendingRowsetGuard after txn_rs_builder init |
572 | 1 | _rowset_id = context.rowset_id; |
573 | | |
574 | 1 | _is_init = true; |
575 | 1 | return Status::OK(); |
576 | 1 | } |
577 | | |
578 | 0 | Status BaseRowsetBuilder::attach_rowset_to_txn(const RowsetSharedPtr& rowset) { |
579 | 0 | if (!is_data_builder()) { |
580 | 0 | return Status::RuntimeError("the rowset isn't allowed to manage txn"); |
581 | 0 | } |
582 | 0 | _attach_rowsets.push_back(rowset); |
583 | 0 | return Status::OK(); |
584 | 0 | } |
585 | | |
586 | 1 | Status BaseRowsetBuilder::attach_pending_rs_guard_to_txn(const RowsetId& rowset_id) { |
587 | 1 | if (!is_data_builder()) { |
588 | 0 | return Status::RuntimeError("the rowset isn't allowed to manage txn"); |
589 | 0 | } |
590 | 1 | _attach_rowset_ids.push_back(rowset_id); |
591 | 1 | return Status::OK(); |
592 | 1 | } |
593 | | |
594 | | } // namespace doris |