Coverage Report

Created: 2026-05-16 12:16

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/rowset_builder.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "storage/rowset_builder.h"
19
20
#include <brpc/controller.h>
21
#include <fmt/format.h>
22
23
#include <filesystem>
24
#include <memory>
25
#include <ostream>
26
#include <string>
27
#include <utility>
28
29
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
30
#include "common/compiler_util.h" // IWYU pragma: keep
31
#include "common/config.h"
32
#include "common/status.h"
33
#include "core/block/block.h"
34
#include "io/fs/file_system.h"
35
#include "io/fs/file_writer.h" // IWYU pragma: keep
36
#include "runtime/memory/global_memory_arbitrator.h"
37
#include "storage/delete/calc_delete_bitmap_executor.h"
38
#include "storage/olap_define.h"
39
#include "storage/partial_update_info.h"
40
#include "storage/rowset/beta_rowset.h"
41
#include "storage/rowset/beta_rowset_writer.h"
42
#include "storage/rowset/group_rowset_writer.h"
43
#include "storage/rowset/pending_rowset_helper.h"
44
#include "storage/rowset/rowset_factory.h"
45
#include "storage/rowset/rowset_meta.h"
46
#include "storage/rowset/rowset_meta_manager.h"
47
#include "storage/rowset/rowset_writer.h"
48
#include "storage/rowset/rowset_writer_context.h"
49
#include "storage/schema_change/schema_change.h"
50
#include "storage/storage_engine.h"
51
#include "storage/tablet/tablet_manager.h"
52
#include "storage/tablet/tablet_meta.h"
53
#include "storage/tablet/tablet_schema.h"
54
#include "storage/tablet_info.h"
55
#include "storage/txn/txn_manager.h"
56
#include "util/brpc_client_cache.h"
57
#include "util/brpc_closure.h"
58
#include "util/debug_points.h"
59
#include "util/mem_info.h"
60
#include "util/stopwatch.hpp"
61
#include "util/time.h"
62
#include "util/trace.h"
63
64
namespace doris {
65
using namespace ErrorCode;
66
67
BaseRowsetBuilder::BaseRowsetBuilder(const WriteRequest& req, RuntimeProfile* profile)
68
279k
        : _req(req), _tablet_schema(std::make_shared<TabletSchema>()) {
69
279k
    if (profile != nullptr) {
70
13.7k
        _init_profile(profile);
71
13.7k
    }
72
279k
}
73
74
RowsetBuilder::RowsetBuilder(StorageEngine& engine, const WriteRequest& req,
75
                             RuntimeProfile* profile)
76
17.2k
        : BaseRowsetBuilder(req, profile), _engine(engine) {}
77
78
RowBinlogRowsetBuilder::RowBinlogRowsetBuilder(StorageEngine& engine, const WriteRequest& req,
79
                                               RuntimeProfile* profile)
80
1
        : RowsetBuilder(engine, req, profile) {}
81
82
13.7k
void BaseRowsetBuilder::_init_profile(RuntimeProfile* profile) {
83
13.7k
    DCHECK(profile != nullptr);
84
13.7k
    if (_req.write_req_type == WriteRequestType::GROUP) {
85
1
        _profile = profile->create_child(fmt::format("GroupRowsetBuilder {}", _req.tablet_id), true,
86
1
                                         true);
87
1
        return;
88
1
    }
89
90
13.7k
    _profile = profile->create_child(
91
13.7k
            fmt::format(
92
13.7k
                    "RowsetBuilder {} {}", _req.tablet_id,
93
13.7k
                    _req.write_req_type == WriteRequestType::ROW_BINLOG ? "binlog<row>" : "data"),
94
13.7k
            true, true);
95
13.7k
    _build_rowset_timer = ADD_TIMER(_profile, "BuildRowsetTime");
96
13.7k
    _submit_delete_bitmap_timer = ADD_TIMER(_profile, "DeleteBitmapSubmitTime");
97
13.7k
    _wait_delete_bitmap_timer = ADD_TIMER(_profile, "DeleteBitmapWaitTime");
98
13.7k
}
99
100
0
void RowsetBuilder::_init_profile(RuntimeProfile* profile) {
101
0
    DCHECK(profile != nullptr);
102
0
    BaseRowsetBuilder::_init_profile(profile);
103
0
    _commit_txn_timer = ADD_TIMER(_profile, "CommitTxnTime");
104
0
}
105
106
279k
BaseRowsetBuilder::~BaseRowsetBuilder() {
107
279k
    if (!_is_init) {
108
91.5k
        return;
109
91.5k
    }
110
111
187k
    if (_calc_delete_bitmap_token != nullptr) {
112
187k
        _calc_delete_bitmap_token->cancel();
113
187k
    }
114
187k
}
115
116
17.2k
RowsetBuilder::~RowsetBuilder() {
117
17.2k
    if (_is_init && !_is_committed) {
118
        // For txn rowset builders, we need to rollback txn when necessary.
119
7
        _garbage_collection(is_data_builder());
120
7
    }
121
17.2k
}
122
123
129k
Tablet* RowsetBuilder::tablet() {
124
129k
    return static_cast<Tablet*>(_tablet.get());
125
129k
}
126
127
7
void RowsetBuilder::_garbage_collection(bool cancel_txn) {
128
7
    Status rollback_status;
129
7
    bool need_clean = true;
130
7
    if (tablet() != nullptr && cancel_txn) {
131
6
        TxnManager* txn_mgr = _engine.txn_manager();
132
6
        rollback_status = txn_mgr->rollback_txn(_req.partition_id, *tablet(), _req.txn_id);
133
6
        need_clean = rollback_status.ok();
134
6
    }
135
    // has to check rollback status, because the rowset maybe committed in this thread and
136
    // published in another thread, then rollback will fail.
137
    // when rollback failed should not delete rowset
138
7
    if (need_clean) {
139
7
        _engine.add_unused_rowset(_rowset);
140
7
        for (auto& rs : _attach_rowsets) {
141
0
            _engine.add_unused_rowset(rs);
142
0
        }
143
7
    }
144
7
}
145
146
57.7k
Status BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_context) {
147
57.7k
    DCHECK(is_data_builder());
148
149
57.7k
    std::lock_guard<std::shared_mutex> lck(tablet()->get_header_lock());
150
57.7k
    _max_version_in_flush_phase = tablet()->max_version_unlocked();
151
57.7k
    std::vector<RowsetSharedPtr> rowset_ptrs;
152
    // tablet is under alter process. The delete bitmap will be calculated after conversion.
153
57.7k
    if (tablet()->tablet_state() == TABLET_NOTREADY) {
154
        // Disable 'partial_update' when the tablet is undergoing a 'schema changing process'
155
30
        if (_req.table_schema_param->is_partial_update()) {
156
0
            return Status::InternalError(
157
0
                    "Unable to do 'partial_update' when "
158
0
                    "the tablet is undergoing a 'schema changing process'");
159
0
        }
160
30
        _rowset_ids->clear();
161
57.6k
    } else {
162
57.6k
        RETURN_IF_ERROR(
163
57.6k
                tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase, _rowset_ids.get()));
164
57.6k
        rowset_ptrs = tablet()->get_rowset_by_ids(_rowset_ids.get());
165
57.6k
    }
166
57.7k
    _delete_bitmap = std::make_shared<DeleteBitmap>(tablet()->tablet_id());
167
57.7k
    mow_context = std::make_shared<MowContext>(_max_version_in_flush_phase, _req.txn_id,
168
57.7k
                                               _rowset_ids, rowset_ptrs, _delete_bitmap);
169
57.7k
    return Status::OK();
170
57.7k
}
171
172
16.2k
Status RowsetBuilder::check_tablet_version_count() {
173
16.2k
    DCHECK(is_data_builder());
174
175
16.2k
    auto max_version_config = _tablet->max_version_config();
176
16.2k
    auto version_count = tablet()->version_count();
177
16.2k
    DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
178
16.2k
                    { version_count = INT_MAX; });
179
    // Trigger TOO MANY VERSION error first
180
16.2k
    if (version_count > max_version_config) {
181
0
        return Status::Error<TOO_MANY_VERSION>(
182
0
                "failed to init rowset builder. version count: {}, exceed limit: {}, "
183
0
                "tablet: {}. Please reduce the frequency of loading data or adjust the "
184
0
                "max_tablet_version_num or time_series_max_tablet_version_num in be.conf to a "
185
0
                "larger value.",
186
0
                version_count, max_version_config, _tablet->tablet_id());
187
0
    }
188
    // (TODO Refrain) Maybe we can use a configurable param instead of hardcoded values '100'.
189
    // max_version_config must > 100, otherwise silent errors will occur.
190
16.2k
    if ((!config::disable_auto_compaction &&
191
16.2k
         !_tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) &&
192
16.2k
        (version_count > max_version_config - 100) &&
193
16.2k
        !GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
194
        // Trigger compaction
195
0
        auto st = _engine.submit_compaction_task(std::static_pointer_cast<Tablet>(tablet_sptr()),
196
0
                                                 CompactionType::CUMULATIVE_COMPACTION, true, true,
197
0
                                                 2);
198
0
        if (!st.ok()) [[unlikely]] {
199
0
            LOG(WARNING) << "failed to trigger compaction, tablet_id=" << _tablet->tablet_id()
200
0
                         << " : " << st;
201
0
        }
202
0
    }
203
16.2k
    return Status::OK();
204
16.2k
}
205
206
16.2k
Status RowsetBuilder::prepare_txn() {
207
16.2k
    DCHECK(is_data_builder());
208
16.2k
    return tablet()->prepare_txn(_req.partition_id, _req.txn_id, _req.load_id, false);
209
16.2k
}
210
211
16.2k
Status RowsetBuilder::init() {
212
16.2k
    RowsetWriterContext context;
213
214
16.2k
    RETURN_IF_ERROR(_init_context_common_fields(context));
215
216
16.2k
    std::shared_ptr<MowContext> mow_context;
217
16.2k
    if (_tablet->enable_unique_key_merge_on_write()) {
218
6.05k
        RETURN_IF_ERROR(init_mow_context(mow_context));
219
6.05k
    }
220
221
16.2k
    RETURN_IF_ERROR(check_tablet_version_count());
222
223
16.2k
    auto version_count = tablet()->version_count() + tablet()->stale_version_count();
224
16.2k
    if (tablet()->avg_rs_meta_serialize_size() * version_count >
225
16.2k
        config::tablet_meta_serialize_size_limit) {
226
0
        return Status::Error<TOO_MANY_VERSION>(
227
0
                "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, "
228
0
                "tablet: {}. Please reduce the frequency of loading data or adjust the "
229
0
                "max_tablet_version_num in be.conf to a larger value.",
230
0
                tablet()->avg_rs_meta_serialize_size() * version_count,
231
0
                config::tablet_meta_serialize_size_limit, _tablet->tablet_id());
232
0
    }
233
234
16.2k
    RETURN_IF_ERROR(prepare_txn());
235
236
16.2k
    DBUG_EXECUTE_IF("BaseRowsetBuilder::init.check_partial_update_column_num", {
237
16.2k
        if (_req.table_schema_param->partial_update_input_columns().size() !=
238
16.2k
            dp->param<int>("column_num")) {
239
16.2k
            return Status::InternalError("partial update input column num wrong!");
240
16.2k
        };
241
16.2k
    })
242
    // build tablet schema in request level
243
16.2k
    RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
244
16.2k
                                                 *_tablet->tablet_schema()));
245
246
16.2k
    context.mow_context = mow_context;
247
248
16.2k
    context.partial_update_info = _partial_update_info;
249
16.2k
    _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false));
250
16.2k
    _rowset_id = context.rowset_id;
251
16.2k
    std::vector<RowsetId> tmp_pending_rowset_ids = {_rowset_id};
252
16.2k
    tmp_pending_rowset_ids.resize(1 + _attach_rowset_ids.size());
253
16.2k
    std::copy(_attach_rowset_ids.begin(), _attach_rowset_ids.end(),
254
16.2k
              tmp_pending_rowset_ids.begin() + 1);
255
16.2k
    _pending_rs_guard = _engine.pending_local_rowsets().add(tmp_pending_rowset_ids);
256
257
16.2k
    _calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor()->create_token();
258
259
16.2k
    _is_init = true;
260
16.2k
    return Status::OK();
261
16.2k
}
262
263
16.2k
Status BaseRowsetBuilder::_init_context_common_fields(RowsetWriterContext& context) {
264
16.2k
    _tablet = DORIS_TRY(ExecEnv::get_tablet(_req.tablet_id));
265
266
16.2k
    context.txn_id = _req.txn_id;
267
16.2k
    context.load_id = _req.load_id;
268
16.2k
    context.db_id = _req.table_schema_param->db_id();
269
16.2k
    context.table_id = _req.table_schema_param->table_id();
270
16.2k
    context.rowset_state = PREPARED;
271
16.2k
    context.segments_overlap = OVERLAPPING;
272
16.2k
    context.tablet_schema = _tablet_schema;
273
16.2k
    context.newest_write_timestamp = UnixSeconds();
274
16.2k
    context.tablet_id = _req.tablet_id;
275
16.2k
    context.index_id = _req.index_id;
276
16.2k
    context.tablet = _tablet;
277
16.2k
    context.enable_segcompaction = true;
278
16.2k
    if (_req.write_req_type == WriteRequestType::ROW_BINLOG || !_attach_rowset_ids.empty()) {
279
2
        context.enable_segcompaction = false;
280
2
    }
281
16.2k
    context.write_type = DataWriteType::TYPE_DIRECT;
282
16.2k
    context.write_file_cache = _req.write_file_cache;
283
284
16.2k
    return Status::OK();
285
16.2k
}
286
287
187k
Status BaseRowsetBuilder::build_rowset() {
288
187k
    std::lock_guard<std::mutex> l(_lock);
289
18.4E
    DCHECK(_is_init) << "rowset builder is supposed be to initialized before "
290
18.4E
                        "build_rowset() being called";
291
292
187k
    SCOPED_TIMER(_build_rowset_timer);
293
    // use rowset meta manager to save meta
294
187k
    RETURN_NOT_OK_STATUS_WITH_WARN(_rowset_writer->build(_rowset), "fail to build rowset");
295
187k
    return Status::OK();
296
187k
}
297
298
1
Status GroupRowsetBuilder::build_rowset() {
299
    // build binlog rowset first, then data rowset
300
    // If data rowset build fails after binlog has built some segments, we rely on the
301
    // RowsetBuilder/StorageEngine garbage-collection paths to clean up built-but-uncommitted files.
302
1
    RETURN_IF_ERROR(_row_binlog_rowset_builder->build_rowset());
303
1
    return _txn_rs_builder->build_rowset();
304
1
}
305
306
187k
Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() {
307
187k
    DCHECK(is_data_builder());
308
187k
    if (!_tablet->enable_unique_key_merge_on_write() || _rowset->num_segments() == 0) {
309
164k
        return Status::OK();
310
164k
    }
311
22.9k
    std::lock_guard<std::mutex> l(_lock);
312
22.9k
    SCOPED_TIMER(_submit_delete_bitmap_timer);
313
22.9k
    if (_partial_update_info && _partial_update_info->is_flexible_partial_update()) {
314
264
        if (_rowset->num_segments() > 1) {
315
            // in flexible partial update, when there are more one segment in one load,
316
            // we need to do alignment process for same keys between segments, we haven't
317
            // implemented it yet and just report an error when encouter this situation
318
0
            return Status::NotSupported(
319
0
                    "too large input data in flexible partial update, Please "
320
0
                    "reduce the amount of data imported in a single load.");
321
0
        }
322
264
    }
323
324
22.9k
    auto* beta_rowset = reinterpret_cast<BetaRowset*>(_rowset.get());
325
22.9k
    std::vector<segment_v2::SegmentSharedPtr> segments;
326
22.9k
    RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
327
22.9k
    if (segments.size() > 1) {
328
        // calculate delete bitmap between segments
329
0
        if (config::enable_calc_delete_bitmap_between_segments_concurrently) {
330
0
            RETURN_IF_ERROR(_calc_delete_bitmap_token->submit(
331
0
                    _tablet, _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap));
332
0
        } else {
333
0
            RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(
334
0
                    _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap));
335
0
        }
336
0
    }
337
338
    // For partial update, we need to fill in the entire row of data, during the calculation
339
    // of the delete bitmap. This operation is resource-intensive, and we need to minimize
340
    // the number of times it occurs. Therefore, we skip this operation here.
341
22.9k
    if (_partial_update_info->is_partial_update()) {
342
        // for partial update, the delete bitmap calculation is done while append_block()
343
        // we print it's summarize logs here before commit.
344
1.61k
        LOG(INFO) << fmt::format(
345
1.61k
                "{} calc delete bitmap summary before commit: tablet({}), txn_id({}), "
346
1.61k
                "rowset_ids({}), cur max_version({}), bitmap num({}), bitmap_cardinality({}), num "
347
1.61k
                "rows updated({}), num rows new added({}), num rows deleted({}), total rows({})",
348
1.61k
                _partial_update_info->partial_update_mode_str(), tablet()->tablet_id(), _req.txn_id,
349
1.61k
                _rowset_ids->size(), rowset_writer()->context().mow_context->max_version,
350
1.61k
                _delete_bitmap->get_delete_bitmap_count(), _delete_bitmap->cardinality(),
351
1.61k
                rowset_writer()->num_rows_updated(), rowset_writer()->num_rows_new_added(),
352
1.61k
                rowset_writer()->num_rows_deleted(), rowset_writer()->num_rows());
353
1.61k
        return Status::OK();
354
1.61k
    }
355
356
22.9k
    LOG(INFO) << "submit calc delete bitmap task to executor, tablet_id: " << tablet()->tablet_id()
357
21.3k
              << ", txn_id: " << _req.txn_id;
358
21.3k
    return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset, *_rowset_ids,
359
21.3k
                                                         _delete_bitmap, segments, _req.txn_id,
360
21.3k
                                                         _calc_delete_bitmap_token.get(), nullptr);
361
22.9k
}
362
363
187k
Status BaseRowsetBuilder::wait_calc_delete_bitmap() {
364
187k
    DCHECK(is_data_builder());
365
187k
    if (!_tablet->enable_unique_key_merge_on_write() || _partial_update_info->is_partial_update()) {
366
133k
        return Status::OK();
367
133k
    }
368
54.1k
    std::lock_guard<std::mutex> l(_lock);
369
54.1k
    SCOPED_TIMER(_wait_delete_bitmap_timer);
370
54.1k
    RETURN_IF_ERROR(_calc_delete_bitmap_token->wait());
371
54.1k
    return Status::OK();
372
54.1k
}
373
374
16.2k
Status RowsetBuilder::commit_txn() {
375
16.2k
    DCHECK(is_data_builder());
376
16.2k
    if (tablet()->enable_unique_key_merge_on_write() &&
377
16.2k
        config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0 &&
378
16.2k
        tablet()->tablet_state() != TABLET_NOTREADY) {
379
2.05k
        auto st = tablet()->check_delete_bitmap_correctness(
380
2.05k
                _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids);
381
2.05k
        if (!st.ok()) {
382
0
            LOG(WARNING) << fmt::format(
383
0
                    "[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] "
384
0
                    "delete bitmap correctness check failed in commit phase!",
385
0
                    _req.tablet_id, _req.txn_id, UniqueId(_req.load_id).to_string(),
386
0
                    _req.partition_id);
387
0
            return st;
388
0
        }
389
2.05k
    }
390
16.2k
    std::lock_guard<std::mutex> l(_lock);
391
16.2k
    SCOPED_TIMER(_commit_txn_timer);
392
393
    // Transfer ownership of `PendingRowsetGuard` to `TxnManager`
394
16.2k
    Status res = _engine.txn_manager()->commit_txn(
395
16.2k
            _req.partition_id, *tablet(), _req.txn_id, _req.load_id, _rowset,
396
16.2k
            std::move(_pending_rs_guard), false, _partial_update_info, &_attach_rowsets);
397
398
16.2k
    if (!res && !res.is<PUSH_TRANSACTION_ALREADY_EXIST>()) {
399
0
        LOG(WARNING) << "Failed to commit txn: " << _req.txn_id
400
0
                     << " for rowset: " << _rowset->rowset_id();
401
0
        return res;
402
0
    }
403
16.2k
    if (_tablet->enable_unique_key_merge_on_write()) {
404
        // no need to update binlog_delvec, it'll be updated in publish phase
405
6.05k
        _engine.txn_manager()->set_txn_related_delete_bitmap(
406
6.05k
                _req.partition_id, _req.txn_id, tablet()->tablet_id(), tablet()->tablet_uid(), true,
407
6.05k
                _delete_bitmap, *_rowset_ids, _partial_update_info);
408
6.05k
    }
409
410
16.2k
    _is_committed = true;
411
16.2k
    return Status::OK();
412
16.2k
}
413
414
0
Status BaseRowsetBuilder::cancel() {
415
0
    std::lock_guard<std::mutex> l(_lock);
416
0
    if (_is_cancelled) {
417
0
        return Status::OK();
418
0
    }
419
0
    if (_calc_delete_bitmap_token != nullptr) {
420
0
        _calc_delete_bitmap_token->cancel();
421
0
    }
422
0
    _is_cancelled = true;
423
0
    return Status::OK();
424
0
}
425
426
Status BaseRowsetBuilder::_build_current_tablet_schema(
427
        int64_t index_id, const OlapTableSchemaParam* table_schema_param,
428
187k
        const TabletSchema& ori_tablet_schema) {
429
187k
    const OlapTableIndexSchema* index_schema = nullptr;
430
187k
    if (_req.write_req_type == WriteRequestType::ROW_BINLOG) {
431
1
        const auto* row_binlog_index_schema = table_schema_param->row_binlog_index_schema();
432
1
        DCHECK(row_binlog_index_schema != nullptr);
433
1
        DCHECK_EQ(row_binlog_index_schema->index_id, index_id);
434
1
        index_schema = row_binlog_index_schema;
435
187k
    } else {
436
203k
        for (const auto* schema : table_schema_param->indexes()) {
437
203k
            if (schema->index_id == index_id) {
438
186k
                index_schema = schema;
439
186k
                break;
440
186k
            }
441
203k
        }
442
187k
    }
443
444
187k
    if (index_schema != nullptr && !index_schema->columns.empty() &&
445
187k
        index_schema->columns[0]->unique_id() >= 0) {
446
186k
        _tablet_schema->shawdow_copy_without_columns(ori_tablet_schema);
447
186k
        _tablet_schema->build_current_tablet_schema(
448
186k
                index_id, cast_set<int32_t>(table_schema_param->version()), index_schema,
449
186k
                ori_tablet_schema);
450
186k
    } else {
451
569
        _tablet_schema->copy_from(ori_tablet_schema);
452
569
    }
453
187k
    if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version()) {
454
        // After schema change, should include extracted column
455
        // For example: a table has two columns, k and v
456
        // After adding a column v2, the schema version increases, max_version_schema needs to be updated.
457
        // _tablet_schema includes k, v, and v2
458
        // if v is a variant, need to add the columns decomposed from the v to the _tablet_schema.
459
1.29k
        if (is_data_builder()) {
460
1.29k
            if (_tablet_schema->num_variant_columns() > 0) {
461
105
                TabletSchemaSPtr max_version_schema = std::make_shared<TabletSchema>();
462
105
                max_version_schema->copy_from(*_tablet_schema);
463
105
                max_version_schema->copy_extracted_columns(ori_tablet_schema);
464
105
                _tablet->update_max_version_schema(max_version_schema);
465
1.19k
            } else {
466
1.19k
                _tablet->update_max_version_schema(_tablet_schema);
467
1.19k
            }
468
1.29k
        }
469
1.29k
    }
470
471
187k
    _tablet_schema->set_table_id(table_schema_param->table_id());
472
187k
    _tablet_schema->set_db_id(table_schema_param->db_id());
473
187k
    if (table_schema_param->is_partial_update()) {
474
3.78k
        _tablet_schema->set_auto_increment_column(table_schema_param->auto_increment_coulumn());
475
3.78k
    }
476
    // set partial update columns info
477
187k
    if (is_data_builder()) {
478
187k
        _partial_update_info = std::make_shared<PartialUpdateInfo>();
479
187k
        RETURN_IF_ERROR(_partial_update_info->init(
480
187k
                tablet()->tablet_id(), _req.txn_id, *_tablet_schema,
481
187k
                table_schema_param->unique_key_update_mode(),
482
187k
                table_schema_param->partial_update_new_key_policy(),
483
187k
                table_schema_param->partial_update_input_columns(),
484
187k
                table_schema_param->is_strict_mode(), table_schema_param->timestamp_ms(),
485
187k
                table_schema_param->nano_seconds(), table_schema_param->timezone(),
486
187k
                table_schema_param->auto_increment_coulumn(),
487
187k
                table_schema_param->sequence_map_col_uid(), _max_version_in_flush_phase));
488
187k
    }
489
187k
    return Status::OK();
490
187k
}
491
492
GroupRowsetBuilder::GroupRowsetBuilder(StorageEngine& engine, const WriteRequest& group_build_req,
493
                                       const WriteRequest& sub_data_req,
494
                                       const WriteRequest& sub_row_binlog_req,
495
                                       RuntimeProfile* profile)
496
1
        : BaseRowsetBuilder(group_build_req, profile) {
497
1
    DCHECK(group_build_req.write_req_type == WriteRequestType::GROUP &&
498
1
           sub_data_req.write_req_type == WriteRequestType::DATA &&
499
1
           sub_row_binlog_req.write_req_type == WriteRequestType::ROW_BINLOG);
500
1
    _row_binlog_rowset_builder =
501
1
            std::make_shared<RowBinlogRowsetBuilder>(engine, sub_row_binlog_req, profile);
502
1
    _txn_rs_builder = std::make_shared<RowsetBuilder>(engine, sub_data_req, profile);
503
1
}
504
505
1
Status GroupRowsetBuilder::init() {
506
    // init binlog builder first so that its rowset id can be added into
507
    // PendingLocalRowsets before txn builder init.
508
1
    RETURN_IF_ERROR(_row_binlog_rowset_builder->init());
509
    // before init txn, need to add all rowset_ids into PendingLocalRowsets.
510
    // see https://github.com/apache/doris/pull/25921
511
1
    RETURN_IF_ERROR(_txn_rs_builder->attach_pending_rs_guard_to_txn(
512
1
            _row_binlog_rowset_builder->rowset_id()));
513
1
    RETURN_IF_ERROR(_txn_rs_builder->init());
514
515
    // Create a GroupRowsetWriter that forwards flush to both underlying
516
    // RowsetWriters.
517
1
    std::unique_ptr<doris::GroupRowsetWriter> group_writer;
518
1
    RETURN_IF_ERROR(RowsetFactory::create_empty_group_rowset_writer(&group_writer));
519
1
    group_writer->set_data_writer(_txn_rs_builder->rowset_writer());
520
1
    group_writer->set_row_binlog_writer(_row_binlog_rowset_builder->rowset_writer());
521
522
1
    {
523
1
        const auto& data_ctx = _txn_rs_builder->rowset_writer()->context();
524
1
        auto& binlog_ctx = const_cast<RowsetWriterContext&>(
525
1
                _row_binlog_rowset_builder->rowset_writer()->context());
526
1
        auto& cfg = binlog_ctx.write_binlog_opt().write_binlog_config();
527
1
        cfg.source.tablet_schema = data_ctx.tablet_schema;
528
1
        cfg.source.partial_update_info = data_ctx.partial_update_info;
529
1
        cfg.source.mow_context = data_ctx.mow_context;
530
1
        cfg.source.is_transient_rowset_writer = data_ctx.is_transient_rowset_writer;
531
1
        cfg.source.source_write_type = data_ctx.write_type;
532
1
    }
533
534
1
    _rowset_writer = std::move(group_writer);
535
1
    _is_init = true;
536
1
    return Status::OK();
537
1
}
538
539
0
Status GroupRowsetBuilder::submit_calc_delete_bitmap_task() {
540
0
    return _txn_rs_builder->submit_calc_delete_bitmap_task();
541
0
}
542
543
0
Status GroupRowsetBuilder::wait_calc_delete_bitmap() {
544
0
    return _txn_rs_builder->wait_calc_delete_bitmap();
545
0
}
546
547
0
Status GroupRowsetBuilder::commit_txn() {
548
    // Attach binlog rowset to txn rowset, so that commit/rollback and
549
    // clean-up are all handled by txn rowset builder.
550
0
    RETURN_IF_ERROR(_txn_rs_builder->attach_rowset_to_txn(_row_binlog_rowset_builder->rowset()));
551
0
    auto st = _txn_rs_builder->commit_txn();
552
0
    if (st.ok()) {
553
        // Avoid RowBinlogRowsetBuilder being cleaned in its base dtor.
554
0
        RETURN_IF_ERROR(_row_binlog_rowset_builder->commit_txn());
555
0
    }
556
0
    return st;
557
0
}
558
559
1
Status RowBinlogRowsetBuilder::init() {
560
1
    RowsetWriterContext context;
561
562
1
    RETURN_IF_ERROR(_init_context_common_fields(context));
563
564
    // build tablet schema in request level
565
1
    RETURN_IF_ERROR(_build_current_tablet_schema(
566
1
            _req.index_id, _req.table_schema_param.get(),
567
1
            *std::dynamic_pointer_cast<Tablet>(_tablet)->row_binlog_tablet_schema()));
568
1
    context.write_binlog_opt().enable = true;
569
570
1
    _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false));
571
    // need to attach PendingRowsetGuard after txn_rs_builder init
572
1
    _rowset_id = context.rowset_id;
573
574
1
    _is_init = true;
575
1
    return Status::OK();
576
1
}
577
578
0
Status BaseRowsetBuilder::attach_rowset_to_txn(const RowsetSharedPtr& rowset) {
579
0
    if (!is_data_builder()) {
580
0
        return Status::RuntimeError("the rowset isn't allowed to manage txn");
581
0
    }
582
0
    _attach_rowsets.push_back(rowset);
583
0
    return Status::OK();
584
0
}
585
586
1
Status BaseRowsetBuilder::attach_pending_rs_guard_to_txn(const RowsetId& rowset_id) {
587
1
    if (!is_data_builder()) {
588
0
        return Status::RuntimeError("the rowset isn't allowed to manage txn");
589
0
    }
590
1
    _attach_rowset_ids.push_back(rowset_id);
591
1
    return Status::OK();
592
1
}
593
594
} // namespace doris