be/src/storage/rowset/rowset_writer.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <gen_cpp/internal_service.pb.h> |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | #include <gen_cpp/types.pb.h> |
23 | | |
24 | | #include <functional> |
25 | | #include <limits> |
26 | | #include <memory> |
27 | | #include <optional> |
28 | | |
29 | | #include "common/factory_creator.h" |
30 | | #include "core/block/block.h" |
31 | | #include "storage/index/index_file_writer.h" |
32 | | #include "storage/olap_define.h" |
33 | | #include "storage/rowset/rowset.h" |
34 | | #include "storage/rowset/rowset_writer_context.h" |
35 | | #include "storage/schema_change/column_mapping.h" |
36 | | #include "storage/tablet/tablet_fwd.h" |
37 | | #include "storage/tablet/tablet_schema.h" |
38 | | |
39 | | namespace doris { |
40 | | |
41 | | struct SegmentStatistics { |
42 | | int64_t row_num; |
43 | | int64_t data_size; |
44 | | int64_t index_size; |
45 | | KeyBoundsPB key_bounds; |
46 | | |
47 | 120k | SegmentStatistics() = default; |
48 | | |
49 | | SegmentStatistics(SegmentStatisticsPB pb) |
50 | 60 | : row_num(pb.row_num()), |
51 | 60 | data_size(pb.data_size()), |
52 | 60 | index_size(pb.index_size()), |
53 | 60 | key_bounds(pb.key_bounds()) {} |
54 | | |
55 | 60 | void to_pb(SegmentStatisticsPB* segstat_pb) const { |
56 | 60 | segstat_pb->set_row_num(row_num); |
57 | 60 | segstat_pb->set_data_size(data_size); |
58 | 60 | segstat_pb->set_index_size(index_size); |
59 | 60 | segstat_pb->mutable_key_bounds()->CopyFrom(key_bounds); |
60 | 60 | } |
61 | | |
62 | 0 | std::string to_string() { |
63 | 0 | std::stringstream ss; |
64 | 0 | ss << "row_num: " << row_num << ", data_size: " << data_size |
65 | 0 | << ", index_size: " << index_size << ", key_bounds: " << key_bounds.ShortDebugString(); |
66 | 0 | return ss.str(); |
67 | 0 | } |
68 | | }; |
69 | | using SegmentStatisticsSharedPtr = std::shared_ptr<SegmentStatistics>; |
70 | | |
71 | | class RowsetWriter { |
72 | | public: |
73 | | static constexpr int32_t MAX_SEGMENT_NUM = std::numeric_limits<int32_t>::max(); |
74 | | |
75 | 198k | RowsetWriter() = default; |
76 | 200k | virtual ~RowsetWriter() = default; |
77 | | |
78 | | virtual Status init(const RowsetWriterContext& rowset_writer_context) = 0; |
79 | | |
80 | 0 | virtual Status add_block(const Block* block) { |
81 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
82 | 0 | "RowsetWriter not support add_block"); |
83 | 0 | } |
84 | | virtual Status add_columns(const Block* block, const std::vector<uint32_t>& col_ids, |
85 | 0 | bool is_key, uint32_t max_rows_per_segment, bool has_cluster_key) { |
86 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
87 | 0 | "RowsetWriter not support add_columns"); |
88 | 0 | } |
89 | | |
90 | | // Precondition: the input `rowset` should have the same type of the rowset we're building |
91 | | virtual Status add_rowset(RowsetSharedPtr rowset) = 0; |
92 | | |
93 | | // Precondition: the input `rowset` should have the same type of the rowset we're building |
94 | | virtual Status add_rowset_for_linked_schema_change(RowsetSharedPtr rowset) = 0; |
95 | | |
96 | | virtual Status create_file_writer(uint32_t segment_id, io::FileWriterPtr& writer, |
97 | 0 | FileType file_type = FileType::SEGMENT_FILE) { |
98 | 0 | return Status::NotSupported("RowsetWriter does not support create_file_writer"); |
99 | 0 | } |
100 | | |
101 | | virtual Status create_index_file_writer(uint32_t segment_id, |
102 | 6.05k | IndexFileWriterPtr* index_file_writer) { |
103 | | // Create file writer for the inverted index format v2. |
104 | 6.05k | io::FileWriterPtr idx_file_v2_ptr; |
105 | 6.05k | if (_context.tablet_schema->get_inverted_index_storage_format() != |
106 | 6.05k | InvertedIndexStorageFormatPB::V1) { |
107 | 6.05k | RETURN_IF_ERROR( |
108 | 6.05k | create_file_writer(segment_id, idx_file_v2_ptr, FileType::INVERTED_INDEX_FILE)); |
109 | 6.05k | } |
110 | 6.05k | std::string segment_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( |
111 | 6.05k | _context.segment_path(segment_id))}; |
112 | | // default to true, only when base compaction, we need to check the config |
113 | 6.05k | bool can_use_ram_dir = true; |
114 | 6.05k | if (_context.compaction_type == ReaderType::READER_BASE_COMPACTION) { |
115 | 57 | can_use_ram_dir = config::inverted_index_ram_dir_enable_when_base_compaction; |
116 | 57 | } |
117 | 6.05k | *index_file_writer = std::make_unique<IndexFileWriter>( |
118 | 6.05k | _context.fs(), segment_prefix, _context.rowset_id.to_string(), segment_id, |
119 | 6.05k | _context.tablet_schema->get_inverted_index_storage_format(), |
120 | 6.05k | std::move(idx_file_v2_ptr), can_use_ram_dir, _context.tablet_id); |
121 | 6.05k | return Status::OK(); |
122 | 6.05k | } |
123 | | |
124 | | // explicit flush all buffered rows into segment file. |
125 | | // note that `add_row` could also trigger flush when certain conditions are met |
126 | | virtual Status flush() = 0; |
127 | 0 | virtual Status flush_columns(bool is_key) { |
128 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
129 | 0 | "RowsetWriter not support flush_columns"); |
130 | 0 | } |
131 | 0 | virtual Status final_flush() { |
132 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
133 | 0 | "RowsetWriter not support final_flush"); |
134 | 0 | } |
135 | | |
136 | 0 | virtual Status flush_memtable(Block* block, int32_t segment_id, int64_t* flush_size) { |
137 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
138 | 0 | "RowsetWriter not support flush_memtable"); |
139 | 0 | } |
140 | | |
141 | 0 | virtual Status flush_single_block(const Block* block) { |
142 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
143 | 0 | "RowsetWriter not support flush_single_block"); |
144 | 0 | } |
145 | | |
146 | 0 | virtual Status flush_single_block(const Block* block, int32_t segment_id) { |
147 | 0 | return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>( |
148 | 0 | "RowsetWriter not support flush_single_block with segment_id"); |
149 | 0 | } |
150 | | |
151 | 0 | virtual Status add_segment(uint32_t segment_id, const SegmentStatistics& segstat) { |
152 | 0 | return Status::NotSupported("RowsetWriter does not support add_segment"); |
153 | 0 | } |
154 | | |
155 | | // finish building and set rowset pointer to the built rowset (guaranteed to be inited). |
156 | | // rowset is invalid if returned Status is not OK |
157 | | virtual Status build(RowsetSharedPtr& rowset) = 0; |
158 | | |
159 | | // For ordered rowset compaction, manual build rowset |
160 | | virtual RowsetSharedPtr manual_build(const RowsetMetaSharedPtr& rowset_meta) = 0; |
161 | | |
162 | | virtual PUniqueId load_id() = 0; |
163 | | |
164 | | virtual Version version() = 0; |
165 | | |
166 | | virtual int64_t num_rows() const = 0; |
167 | | |
168 | | virtual int64_t num_rows_updated() const = 0; |
169 | | virtual int64_t num_rows_deleted() const = 0; |
170 | | virtual int64_t num_rows_new_added() const = 0; |
171 | | virtual int64_t num_rows_filtered() const = 0; |
172 | | |
173 | | virtual RowsetId rowset_id() = 0; |
174 | | |
175 | | virtual RowsetTypePB type() const = 0; |
176 | | |
177 | 0 | virtual Status get_segment_num_rows(std::vector<uint32_t>* segment_num_rows) const { |
178 | 0 | return Status::NotSupported("to be implemented"); |
179 | 0 | } |
180 | | |
181 | | virtual Result<int32_t> allocate_segment_id() = 0; |
182 | | |
183 | | // Return the next segment id to be allocated without advancing internal state. |
184 | | // NOTE: This value equals the one that would be returned by the next |
185 | | // `allocate_segment_id()` call. |
186 | | virtual int32_t get_allocated_segment_id() = 0; |
187 | | |
188 | | // Set the first physical segment id and the maximum number of ids this writer may allocate. |
189 | | // The default maximum preserves the original unbounded allocation behavior. |
190 | 0 | virtual void set_segment_start_id(int32_t start_seg_id, int32_t max_seg_num = MAX_SEGMENT_NUM) { |
191 | 0 | throw Exception(Status::FatalError("not supported!")); |
192 | 0 | } |
193 | | |
194 | 0 | virtual Status force_rollback() { |
195 | 0 | return Status::NotSupported("RowsetWriter::force_rollback not supported"); |
196 | 0 | } |
197 | | |
198 | 0 | virtual int64_t delete_bitmap_ns() { return 0; } |
199 | | |
200 | 0 | virtual int64_t segment_writer_ns() { return 0; } |
201 | | |
202 | | virtual std::shared_ptr<PartialUpdateInfo> get_partial_update_info() = 0; |
203 | | |
204 | | virtual bool is_partial_update() = 0; |
205 | | |
206 | 188k | const RowsetWriterContext& context() { return _context; } |
207 | | |
208 | 495k | const RowsetMetaSharedPtr& rowset_meta() { return _rowset_meta; } |
209 | | |
210 | | private: |
211 | | DISALLOW_COPY_AND_ASSIGN(RowsetWriter); |
212 | | |
213 | | protected: |
214 | | RowsetWriterContext _context; |
215 | | RowsetMetaSharedPtr _rowset_meta; |
216 | | }; |
217 | | |
218 | | } // namespace doris |