Coverage Report

Created: 2026-08-26 16:25

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/rowset/rowset_writer.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/internal_service.pb.h>
21
#include <gen_cpp/olap_file.pb.h>
22
#include <gen_cpp/types.pb.h>
23
24
#include <functional>
25
#include <limits>
26
#include <memory>
27
#include <optional>
28
29
#include "common/factory_creator.h"
30
#include "core/block/block.h"
31
#include "storage/index/index_file_writer.h"
32
#include "storage/olap_define.h"
33
#include "storage/rowset/rowset.h"
34
#include "storage/rowset/rowset_writer_context.h"
35
#include "storage/schema_change/column_mapping.h"
36
#include "storage/tablet/tablet_fwd.h"
37
#include "storage/tablet/tablet_schema.h"
38
39
namespace doris {
40
41
struct SegmentStatistics {
42
    int64_t row_num;
43
    int64_t data_size;
44
    int64_t index_size;
45
    KeyBoundsPB key_bounds;
46
47
120k
    SegmentStatistics() = default;
48
49
    SegmentStatistics(SegmentStatisticsPB pb)
50
60
            : row_num(pb.row_num()),
51
60
              data_size(pb.data_size()),
52
60
              index_size(pb.index_size()),
53
60
              key_bounds(pb.key_bounds()) {}
54
55
60
    void to_pb(SegmentStatisticsPB* segstat_pb) const {
56
60
        segstat_pb->set_row_num(row_num);
57
60
        segstat_pb->set_data_size(data_size);
58
60
        segstat_pb->set_index_size(index_size);
59
60
        segstat_pb->mutable_key_bounds()->CopyFrom(key_bounds);
60
60
    }
61
62
0
    std::string to_string() {
63
0
        std::stringstream ss;
64
0
        ss << "row_num: " << row_num << ", data_size: " << data_size
65
0
           << ", index_size: " << index_size << ", key_bounds: " << key_bounds.ShortDebugString();
66
0
        return ss.str();
67
0
    }
68
};
69
using SegmentStatisticsSharedPtr = std::shared_ptr<SegmentStatistics>;
70
71
class RowsetWriter {
72
public:
73
    static constexpr int32_t MAX_SEGMENT_NUM = std::numeric_limits<int32_t>::max();
74
75
199k
    RowsetWriter() = default;
76
201k
    virtual ~RowsetWriter() = default;
77
78
    virtual Status init(const RowsetWriterContext& rowset_writer_context) = 0;
79
80
0
    virtual Status add_block(const Block* block) {
81
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
82
0
                "RowsetWriter not support add_block");
83
0
    }
84
    virtual Status add_columns(const Block* block, const std::vector<uint32_t>& col_ids,
85
0
                               bool is_key, uint32_t max_rows_per_segment, bool has_cluster_key) {
86
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
87
0
                "RowsetWriter not support add_columns");
88
0
    }
89
90
    // Precondition: the input `rowset` should have the same type of the rowset we're building
91
    virtual Status add_rowset(RowsetSharedPtr rowset) = 0;
92
93
    // Precondition: the input `rowset` should have the same type of the rowset we're building
94
    virtual Status add_rowset_for_linked_schema_change(RowsetSharedPtr rowset) = 0;
95
96
    virtual Status create_file_writer(uint32_t segment_id, io::FileWriterPtr& writer,
97
0
                                      FileType file_type = FileType::SEGMENT_FILE) {
98
0
        return Status::NotSupported("RowsetWriter does not support create_file_writer");
99
0
    }
100
101
    virtual Status create_index_file_writer(uint32_t segment_id,
102
6.08k
                                            IndexFileWriterPtr* index_file_writer) {
103
        // Create file writer for the inverted index format v2.
104
6.08k
        io::FileWriterPtr idx_file_v2_ptr;
105
6.08k
        if (_context.tablet_schema->get_inverted_index_storage_format() !=
106
6.08k
            InvertedIndexStorageFormatPB::V1) {
107
6.06k
            RETURN_IF_ERROR(
108
6.06k
                    create_file_writer(segment_id, idx_file_v2_ptr, FileType::INVERTED_INDEX_FILE));
109
6.06k
        }
110
6.08k
        std::string segment_prefix {InvertedIndexDescriptor::get_index_file_path_prefix(
111
6.08k
                _context.segment_path(segment_id))};
112
        // default to true, only when base compaction, we need to check the config
113
6.08k
        bool can_use_ram_dir = true;
114
6.08k
        if (_context.compaction_type == ReaderType::READER_BASE_COMPACTION) {
115
61
            can_use_ram_dir = config::inverted_index_ram_dir_enable_when_base_compaction;
116
61
        }
117
6.08k
        *index_file_writer = std::make_unique<IndexFileWriter>(
118
6.08k
                _context.fs(), segment_prefix, _context.rowset_id.to_string(), segment_id,
119
6.08k
                _context.tablet_schema->get_inverted_index_storage_format(),
120
6.08k
                std::move(idx_file_v2_ptr), can_use_ram_dir, _context.tablet_id);
121
6.08k
        return Status::OK();
122
6.08k
    }
123
124
    // explicit flush all buffered rows into segment file.
125
    // note that `add_row` could also trigger flush when certain conditions are met
126
    virtual Status flush() = 0;
127
0
    virtual Status flush_columns(bool is_key) {
128
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
129
0
                "RowsetWriter not support flush_columns");
130
0
    }
131
0
    virtual Status final_flush() {
132
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
133
0
                "RowsetWriter not support final_flush");
134
0
    }
135
136
0
    virtual Status flush_memtable(Block* block, int32_t segment_id, int64_t* flush_size) {
137
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
138
0
                "RowsetWriter not support flush_memtable");
139
0
    }
140
141
0
    virtual Status flush_single_block(const Block* block) {
142
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
143
0
                "RowsetWriter not support flush_single_block");
144
0
    }
145
146
0
    virtual Status flush_single_block(const Block* block, int32_t segment_id) {
147
0
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
148
0
                "RowsetWriter not support flush_single_block with segment_id");
149
0
    }
150
151
0
    virtual Status add_segment(uint32_t segment_id, const SegmentStatistics& segstat) {
152
0
        return Status::NotSupported("RowsetWriter does not support add_segment");
153
0
    }
154
155
    // finish building and set rowset pointer to the built rowset (guaranteed to be inited).
156
    // rowset is invalid if returned Status is not OK
157
    virtual Status build(RowsetSharedPtr& rowset) = 0;
158
159
    // For ordered rowset compaction, manual build rowset
160
    virtual RowsetSharedPtr manual_build(const RowsetMetaSharedPtr& rowset_meta) = 0;
161
162
    virtual PUniqueId load_id() = 0;
163
164
    virtual Version version() = 0;
165
166
    virtual int64_t num_rows() const = 0;
167
168
    virtual int64_t num_rows_updated() const = 0;
169
    virtual int64_t num_rows_deleted() const = 0;
170
    virtual int64_t num_rows_new_added() const = 0;
171
    virtual int64_t num_rows_filtered() const = 0;
172
173
    virtual RowsetId rowset_id() = 0;
174
175
    virtual RowsetTypePB type() const = 0;
176
177
0
    virtual Status get_segment_num_rows(std::vector<uint32_t>* segment_num_rows) const {
178
0
        return Status::NotSupported("to be implemented");
179
0
    }
180
181
    virtual Result<int32_t> allocate_segment_id() = 0;
182
183
    // Return the next segment id to be allocated without advancing internal state.
184
    // NOTE: This value equals the one that would be returned by the next
185
    // `allocate_segment_id()` call.
186
    virtual int32_t get_allocated_segment_id() = 0;
187
188
    // Set the first physical segment id and the maximum number of ids this writer may allocate.
189
    // The default maximum preserves the original unbounded allocation behavior.
190
0
    virtual void set_segment_start_id(int32_t start_seg_id, int32_t max_seg_num = MAX_SEGMENT_NUM) {
191
0
        throw Exception(Status::FatalError("not supported!"));
192
0
    }
193
194
0
    virtual Status force_rollback() {
195
0
        return Status::NotSupported("RowsetWriter::force_rollback not supported");
196
0
    }
197
198
0
    virtual int64_t delete_bitmap_ns() { return 0; }
199
200
0
    virtual int64_t segment_writer_ns() { return 0; }
201
202
    virtual std::shared_ptr<PartialUpdateInfo> get_partial_update_info() = 0;
203
204
    virtual bool is_partial_update() = 0;
205
206
189k
    const RowsetWriterContext& context() { return _context; }
207
208
498k
    const RowsetMetaSharedPtr& rowset_meta() { return _rowset_meta; }
209
210
private:
211
    DISALLOW_COPY_AND_ASSIGN(RowsetWriter);
212
213
protected:
214
    RowsetWriterContext _context;
215
    RowsetMetaSharedPtr _rowset_meta;
216
};
217
218
} // namespace doris