be/src/storage/tablet/tablet_reader.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "storage/tablet/tablet_reader.h" |
19 | | |
20 | | #include <gen_cpp/olap_file.pb.h> |
21 | | #include <gen_cpp/segment_v2.pb.h> |
22 | | #include <thrift/protocol/TDebugProtocol.h> |
23 | | |
24 | | #include <algorithm> |
25 | | #include <functional> |
26 | | #include <iterator> |
27 | | #include <memory> |
28 | | #include <numeric> |
29 | | #include <ostream> |
30 | | #include <shared_mutex> |
31 | | |
32 | | #include "common/compiler_util.h" // IWYU pragma: keep |
33 | | #include "common/config.h" |
34 | | #include "common/exception.h" |
35 | | #include "common/logging.h" |
36 | | #include "common/status.h" |
37 | | #include "core/arena.h" |
38 | | #include "core/block/block.h" |
39 | | #include "exec/common/variant_util.h" |
40 | | #include "exprs/bitmapfilter_predicate.h" |
41 | | #include "exprs/bloom_filter_func.h" |
42 | | #include "exprs/create_predicate_function.h" |
43 | | #include "exprs/hybrid_set.h" |
44 | | #include "runtime/query_context.h" |
45 | | #include "runtime/runtime_predicate.h" |
46 | | #include "runtime/runtime_state.h" |
47 | | #include "storage/index/bloom_filter/bloom_filter.h" |
48 | | #include "storage/itoken_extractor.h" |
49 | | #include "storage/olap_common.h" |
50 | | #include "storage/olap_define.h" |
51 | | #include "storage/predicate/column_predicate.h" |
52 | | #include "storage/predicate/like_column_predicate.h" |
53 | | #include "storage/predicate/predicate_creator.h" |
54 | | #include "storage/row_cursor.h" |
55 | | #include "storage/schema.h" |
56 | | #include "storage/tablet/tablet.h" |
57 | | #include "storage/tablet/tablet_meta.h" |
58 | | #include "storage/tablet/tablet_schema.h" |
59 | | |
60 | | namespace doris { |
61 | | using namespace ErrorCode; |
62 | | |
63 | 1.22M | void TabletReader::ReaderParams::check_validation() const { |
64 | 1.22M | if (UNLIKELY(version.first == -1 && is_segcompaction == false)) { |
65 | 0 | throw Exception(Status::FatalError("version is not set. tablet={}", tablet->tablet_id())); |
66 | 0 | } |
67 | 1.22M | } |
68 | | |
69 | 1.22M | Status TabletReader::init(const ReaderParams& read_params) { |
70 | 1.22M | Status res = _init_params(read_params); |
71 | 1.22M | if (!res.ok()) { |
72 | 0 | LOG(WARNING) << "fail to init reader when init params. res:" << res |
73 | 0 | << ", tablet_id:" << read_params.tablet->tablet_id() |
74 | 0 | << ", schema_hash:" << read_params.tablet->schema_hash() |
75 | 0 | << ", reader type:" << int(read_params.reader_type) |
76 | 0 | << ", version:" << read_params.version; |
77 | 0 | } |
78 | 1.22M | return res; |
79 | 1.22M | } |
80 | | |
81 | 354k | Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { |
82 | 354k | SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns); |
83 | 354k | if (read_params.rs_splits.empty()) { |
84 | 0 | return Status::InternalError("fail to acquire data sources. tablet={}", |
85 | 0 | _tablet->tablet_id()); |
86 | 0 | } |
87 | | |
88 | 354k | bool eof = false; |
89 | 354k | bool is_lower_key_included = _keys_param.start_key_include; |
90 | 354k | bool is_upper_key_included = _keys_param.end_key_include; |
91 | | |
92 | 689k | for (int i = 0; i < _keys_param.start_keys.size(); ++i) { |
93 | | // lower bound |
94 | 334k | RowCursor& start_key = _keys_param.start_keys[i]; |
95 | 334k | RowCursor& end_key = _keys_param.end_keys[i]; |
96 | | |
97 | 334k | if (!is_lower_key_included) { |
98 | 0 | if (compare_row_key(start_key, end_key) >= 0) { |
99 | 0 | VLOG_NOTICE << "return EOF when lower key not include" |
100 | 0 | << ", start_key=" << start_key.to_string() |
101 | 0 | << ", end_key=" << end_key.to_string(); |
102 | 0 | eof = true; |
103 | 0 | break; |
104 | 0 | } |
105 | 334k | } else { |
106 | 334k | if (compare_row_key(start_key, end_key) > 0) { |
107 | 0 | VLOG_NOTICE << "return EOF when lower key include=" |
108 | 0 | << ", start_key=" << start_key.to_string() |
109 | 0 | << ", end_key=" << end_key.to_string(); |
110 | 0 | eof = true; |
111 | 0 | break; |
112 | 0 | } |
113 | 334k | } |
114 | | |
115 | 334k | _is_lower_keys_included.push_back(is_lower_key_included); |
116 | 334k | _is_upper_keys_included.push_back(is_upper_key_included); |
117 | 334k | } |
118 | | |
119 | 354k | if (eof) { |
120 | 0 | return Status::EndOfFile("reach end of scan range. tablet={}", _tablet->tablet_id()); |
121 | 0 | } |
122 | | |
123 | 354k | bool need_ordered_result = true; |
124 | 354k | if (read_params.reader_type == ReaderType::READER_QUERY || |
125 | 354k | read_params.reader_type == ReaderType::READER_BINLOG) { |
126 | 348k | if (_tablet_schema->keys_type() == DUP_KEYS) { |
127 | | // duplicated keys are allowed, no need to merge sort keys in rowset |
128 | 27.1k | need_ordered_result = false; |
129 | 27.1k | } |
130 | 348k | if (_tablet_schema->keys_type() == UNIQUE_KEYS && |
131 | 348k | _tablet->enable_unique_key_merge_on_write()) { |
132 | | // unique keys with merge on write, no need to merge sort keys in rowset |
133 | 320k | need_ordered_result = false; |
134 | 320k | } |
135 | 348k | if (_aggregation) { |
136 | | // compute engine will aggregate rows with the same key, |
137 | | // it's ok for rowset to return unordered result |
138 | 347k | need_ordered_result = false; |
139 | 347k | } |
140 | | |
141 | 348k | if (_direct_mode) { |
142 | | // direct mode indicates that the storage layer does not need to merge, |
143 | | // it's ok for rowset to return unordered result |
144 | 347k | need_ordered_result = false; |
145 | 347k | } |
146 | | |
147 | 348k | if (read_params.read_orderby_key) { |
148 | 0 | need_ordered_result = true; |
149 | 0 | } |
150 | 348k | } |
151 | | |
152 | 354k | _reader_context.reader_type = read_params.reader_type; |
153 | 354k | _reader_context.version = read_params.version; |
154 | 354k | _reader_context.tablet_schema = _tablet_schema; |
155 | 354k | _reader_context.need_ordered_result = need_ordered_result; |
156 | 354k | _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids; |
157 | 354k | _reader_context.topn_filter_target_node_id = read_params.topn_filter_target_node_id; |
158 | 354k | _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; |
159 | 354k | _reader_context.read_orderby_key_limit = read_params.read_orderby_key_limit; |
160 | 354k | _reader_context.return_columns = &_return_columns; |
161 | 354k | _reader_context.read_orderby_key_columns = |
162 | 354k | !_orderby_key_columns.empty() ? &_orderby_key_columns : nullptr; |
163 | 354k | _reader_context.predicates = &_col_predicates; |
164 | 354k | _reader_context.value_predicates = &_value_col_predicates; |
165 | 354k | _reader_context.lower_bound_keys = &_keys_param.start_keys; |
166 | 354k | _reader_context.is_lower_keys_included = &_is_lower_keys_included; |
167 | 354k | _reader_context.upper_bound_keys = &_keys_param.end_keys; |
168 | 354k | _reader_context.is_upper_keys_included = &_is_upper_keys_included; |
169 | 354k | _reader_context.delete_handler = &_delete_handler; |
170 | 354k | _reader_context.stats = &_stats; |
171 | 354k | _reader_context.use_page_cache = read_params.use_page_cache; |
172 | 354k | _reader_context.sequence_id_idx = _sequence_col_idx; |
173 | 354k | _reader_context.is_unique = tablet()->keys_type() == UNIQUE_KEYS; |
174 | 354k | _reader_context.merged_rows = &_merged_rows; |
175 | 354k | _reader_context.delete_bitmap = read_params.delete_bitmap; |
176 | 354k | _reader_context.enable_unique_key_merge_on_write = tablet()->enable_unique_key_merge_on_write(); |
177 | 354k | _reader_context.enable_mor_value_predicate_pushdown = |
178 | 354k | read_params.enable_mor_value_predicate_pushdown; |
179 | 354k | _reader_context.record_rowids = read_params.record_rowids; |
180 | 354k | _reader_context.rowid_conversion = read_params.rowid_conversion; |
181 | 354k | _reader_context.is_key_column_group = read_params.is_key_column_group; |
182 | 354k | _reader_context.remaining_conjunct_roots = read_params.remaining_conjunct_roots; |
183 | 354k | _reader_context.common_expr_ctxs_push_down = read_params.common_expr_ctxs_push_down; |
184 | 354k | _reader_context.output_columns = &read_params.output_columns; |
185 | 354k | _reader_context.push_down_agg_type_opt = read_params.push_down_agg_type_opt; |
186 | 354k | _reader_context.ttl_seconds = _tablet->ttl_seconds(); |
187 | 354k | _reader_context.score_runtime = read_params.score_runtime; |
188 | 354k | _reader_context.collection_statistics = read_params.collection_statistics; |
189 | | |
190 | 354k | _reader_context.virtual_column_exprs = read_params.virtual_column_exprs; |
191 | 354k | _reader_context.vir_cid_to_idx_in_block = read_params.vir_cid_to_idx_in_block; |
192 | 354k | _reader_context.vir_col_idx_to_type = read_params.vir_col_idx_to_type; |
193 | 354k | _reader_context.ann_topn_runtime = read_params.ann_topn_runtime; |
194 | | |
195 | 354k | _reader_context.condition_cache_digest = read_params.condition_cache_digest; |
196 | 354k | _reader_context.all_access_paths = read_params.all_access_paths; |
197 | 354k | _reader_context.predicate_access_paths = read_params.predicate_access_paths; |
198 | | |
199 | | // Propagate general read limit for DUP_KEYS and UNIQUE_KEYS with MOW |
200 | 354k | _reader_context.general_read_limit = read_params.general_read_limit; |
201 | | |
202 | | // Preserve the original requested output layout so BlockReader can map expanded storage |
203 | | // columns (for non-direct AGG/UNIQUE paths) back to the final output block. |
204 | 354k | _reader_context.origin_return_columns = read_params.origin_return_columns; |
205 | | |
206 | 354k | return Status::OK(); |
207 | 354k | } |
208 | | |
209 | 352 | TabletColumn TabletReader::materialize_column(const TabletColumn& orig) { |
210 | 352 | if (!orig.is_variant_type()) { |
211 | 352 | return orig; |
212 | 352 | } |
213 | 0 | TabletColumn column_with_cast_type = orig; |
214 | 0 | auto cast_type = _reader_context.target_cast_type_for_variants.at(orig.name()); |
215 | 0 | return variant_util::get_column_by_type(cast_type, orig.name(), |
216 | 0 | { |
217 | 0 | .unique_id = orig.unique_id(), |
218 | 0 | .parent_unique_id = orig.parent_unique_id(), |
219 | 0 | .path_info = *orig.path_info_ptr(), |
220 | 0 | }); |
221 | 352 | } |
222 | | |
223 | 1.22M | Status TabletReader::_init_params(const ReaderParams& read_params) { |
224 | 1.22M | read_params.check_validation(); |
225 | | |
226 | 1.22M | _direct_mode = read_params.direct_mode; |
227 | 1.22M | _aggregation = read_params.aggregation; |
228 | 1.22M | _reader_type = read_params.reader_type; |
229 | 1.22M | _tablet = read_params.tablet; |
230 | 1.22M | _tablet_schema = read_params.tablet_schema; |
231 | 1.22M | _reader_context.runtime_state = read_params.runtime_state; |
232 | 1.22M | _reader_context.target_cast_type_for_variants = read_params.target_cast_type_for_variants; |
233 | | |
234 | 1.22M | RETURN_IF_ERROR(_init_conditions_param(read_params)); |
235 | | |
236 | 1.22M | Status res = _init_delete_condition(read_params); |
237 | 1.22M | if (!res.ok()) { |
238 | 0 | LOG(WARNING) << "fail to init delete param. res = " << res; |
239 | 0 | return res; |
240 | 0 | } |
241 | | |
242 | 1.22M | res = _init_return_columns(read_params); |
243 | 1.22M | if (!res.ok()) { |
244 | 0 | LOG(WARNING) << "fail to init return columns. res = " << res; |
245 | 0 | return res; |
246 | 0 | } |
247 | | |
248 | 1.22M | res = _init_keys_param(read_params); |
249 | 1.22M | if (!res.ok()) { |
250 | 0 | LOG(WARNING) << "fail to init keys param. res=" << res; |
251 | 0 | return res; |
252 | 0 | } |
253 | 1.22M | res = _init_orderby_keys_param(read_params); |
254 | 1.22M | if (!res.ok()) { |
255 | 0 | LOG(WARNING) << "fail to init orderby keys param. res=" << res; |
256 | 0 | return res; |
257 | 0 | } |
258 | 1.22M | if (_tablet_schema->has_sequence_col()) { |
259 | 4.88k | auto sequence_col_idx = _tablet_schema->sequence_col_idx(); |
260 | 4.88k | DCHECK_NE(sequence_col_idx, -1); |
261 | 18.3k | for (auto col : _return_columns) { |
262 | | // query has sequence col |
263 | 18.3k | if (col == sequence_col_idx) { |
264 | 411 | _sequence_col_idx = sequence_col_idx; |
265 | 411 | break; |
266 | 411 | } |
267 | 18.3k | } |
268 | 4.88k | } |
269 | | |
270 | 1.22M | return res; |
271 | 1.22M | } |
272 | | |
273 | 354k | Status TabletReader::_init_return_columns(const ReaderParams& read_params) { |
274 | 354k | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns); |
275 | 354k | if (read_params.reader_type == ReaderType::READER_QUERY || |
276 | 354k | read_params.reader_type == ReaderType::READER_BINLOG) { |
277 | 348k | _return_columns = read_params.return_columns; |
278 | 348k | _tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set; |
279 | 4.94M | for (auto id : read_params.return_columns) { |
280 | 4.94M | if (_tablet_schema->column(id).is_key()) { |
281 | 2.15M | _key_cids.push_back(id); |
282 | 2.78M | } else { |
283 | 2.78M | _value_cids.push_back(id); |
284 | 2.78M | } |
285 | 4.94M | } |
286 | 348k | } else if (read_params.return_columns.empty()) { |
287 | 0 | for (uint32_t i = 0; i < _tablet_schema->num_columns(); ++i) { |
288 | 0 | _return_columns.push_back(i); |
289 | 0 | if (_tablet_schema->column(i).is_key()) { |
290 | 0 | _key_cids.push_back(i); |
291 | 0 | } else { |
292 | 0 | _value_cids.push_back(i); |
293 | 0 | } |
294 | 0 | } |
295 | 0 | VLOG_NOTICE << "return column is empty, using full column as default."; |
296 | 6.03k | } else if ((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION || |
297 | 6.03k | read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || |
298 | 6.03k | read_params.reader_type == ReaderType::READER_BASE_COMPACTION || |
299 | 6.03k | read_params.reader_type == ReaderType::READER_FULL_COMPACTION || |
300 | 6.03k | read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || |
301 | 6.03k | read_params.reader_type == ReaderType::READER_ALTER_TABLE) && |
302 | 6.18k | !read_params.return_columns.empty()) { |
303 | 6.18k | _return_columns = read_params.return_columns; |
304 | 21.5k | for (auto id : read_params.return_columns) { |
305 | 21.5k | if (_tablet_schema->column(id).is_key()) { |
306 | 6.61k | _key_cids.push_back(id); |
307 | 14.9k | } else { |
308 | 14.9k | _value_cids.push_back(id); |
309 | 14.9k | } |
310 | 21.5k | } |
311 | 18.4E | } else if (read_params.reader_type == ReaderType::READER_CHECKSUM) { |
312 | 0 | _return_columns = read_params.return_columns; |
313 | 0 | for (auto id : read_params.return_columns) { |
314 | 0 | if (_tablet_schema->column(id).is_key()) { |
315 | 0 | _key_cids.push_back(id); |
316 | 0 | } else { |
317 | 0 | _value_cids.push_back(id); |
318 | 0 | } |
319 | 0 | } |
320 | 18.4E | } else { |
321 | 18.4E | return Status::Error<INVALID_ARGUMENT>( |
322 | 18.4E | "fail to init return columns. reader_type={}, return_columns_size={}", |
323 | 18.4E | int(read_params.reader_type), read_params.return_columns.size()); |
324 | 18.4E | } |
325 | | |
326 | 354k | std::sort(_key_cids.begin(), _key_cids.end(), std::greater<>()); |
327 | | |
328 | 354k | return Status::OK(); |
329 | 354k | } |
330 | | |
331 | 1.22M | Status TabletReader::_init_keys_param(const ReaderParams& read_params) { |
332 | 1.22M | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns); |
333 | 1.22M | if (read_params.start_key.empty()) { |
334 | 248k | return Status::OK(); |
335 | 248k | } |
336 | | |
337 | 979k | _keys_param.start_key_include = read_params.start_key_include; |
338 | 979k | _keys_param.end_key_include = read_params.end_key_include; |
339 | | |
340 | 979k | size_t start_key_size = read_params.start_key.size(); |
341 | | //_keys_param.start_keys.resize(start_key_size); |
342 | 979k | std::vector<RowCursor>(start_key_size).swap(_keys_param.start_keys); |
343 | | |
344 | 979k | size_t scan_key_size = read_params.start_key.front().size(); |
345 | 979k | if (scan_key_size > _tablet_schema->num_columns()) { |
346 | 0 | return Status::Error<INVALID_ARGUMENT>( |
347 | 0 | "Input param are invalid. Column count is bigger than num_columns of schema. " |
348 | 0 | "column_count={}, schema.num_columns={}", |
349 | 0 | scan_key_size, _tablet_schema->num_columns()); |
350 | 0 | } |
351 | | |
352 | 979k | std::vector<uint32_t> columns(scan_key_size); |
353 | 979k | std::iota(columns.begin(), columns.end(), 0); |
354 | | |
355 | 979k | std::shared_ptr<Schema> schema = std::make_shared<Schema>(_tablet_schema->columns(), columns); |
356 | | |
357 | 2.64M | for (size_t i = 0; i < start_key_size; ++i) { |
358 | 1.66M | if (read_params.start_key[i].size() != scan_key_size) { |
359 | 0 | return Status::Error<INVALID_ARGUMENT>( |
360 | 0 | "The start_key.at({}).size={}, not equals the scan_key_size={}", i, |
361 | 0 | read_params.start_key[i].size(), scan_key_size); |
362 | 0 | } |
363 | | |
364 | 1.66M | Status res = |
365 | 1.66M | _keys_param.start_keys[i].init(_tablet_schema, read_params.start_key[i], schema); |
366 | 1.66M | if (!res.ok()) { |
367 | 0 | LOG(WARNING) << "fail to init row cursor. res = " << res; |
368 | 0 | return res; |
369 | 0 | } |
370 | 1.66M | } |
371 | | |
372 | 979k | size_t end_key_size = read_params.end_key.size(); |
373 | | //_keys_param.end_keys.resize(end_key_size); |
374 | 979k | std::vector<RowCursor>(end_key_size).swap(_keys_param.end_keys); |
375 | 2.64M | for (size_t i = 0; i < end_key_size; ++i) { |
376 | 1.67M | if (read_params.end_key[i].size() != scan_key_size) { |
377 | 0 | return Status::Error<INVALID_ARGUMENT>( |
378 | 0 | "The end_key.at({}).size={}, not equals the scan_key_size={}", i, |
379 | 0 | read_params.end_key[i].size(), scan_key_size); |
380 | 0 | } |
381 | | |
382 | 1.67M | Status res = _keys_param.end_keys[i].init(_tablet_schema, read_params.end_key[i], schema); |
383 | 1.67M | if (!res.ok()) { |
384 | 0 | LOG(WARNING) << "fail to init row cursor. res = " << res; |
385 | 0 | return res; |
386 | 0 | } |
387 | 1.67M | } |
388 | | |
389 | | //TODO:check the valid of start_key and end_key.(eg. start_key <= end_key) |
390 | | |
391 | 979k | return Status::OK(); |
392 | 979k | } |
393 | | |
394 | 1.22M | Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) { |
395 | 1.22M | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns); |
396 | | // UNIQUE_KEYS will compare all keys as before |
397 | 1.22M | if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS && |
398 | 1.18M | _tablet->enable_unique_key_merge_on_write())) { |
399 | 1.18M | if (!_tablet_schema->cluster_key_uids().empty()) { |
400 | 5.65k | if (read_params.read_orderby_key_num_prefix_columns > |
401 | 5.65k | _tablet_schema->cluster_key_uids().size()) { |
402 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
403 | 0 | "read_orderby_key_num_prefix_columns={} > cluster_keys.size()={}", |
404 | 0 | read_params.read_orderby_key_num_prefix_columns, |
405 | 0 | _tablet_schema->cluster_key_uids().size()); |
406 | 0 | } |
407 | 5.66k | for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { |
408 | 10 | auto cid = _tablet_schema->cluster_key_uids()[i]; |
409 | 10 | auto index = _tablet_schema->field_index(cid); |
410 | 10 | if (index < 0) { |
411 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
412 | 0 | "could not find cluster key column with unique_id=" + |
413 | 0 | std::to_string(cid) + |
414 | 0 | " in tablet schema, tablet_id=" + std::to_string(_tablet->tablet_id())); |
415 | 0 | } |
416 | 20 | for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { |
417 | 20 | if (_return_columns[idx] == index) { |
418 | 10 | _orderby_key_columns.push_back(idx); |
419 | 10 | break; |
420 | 10 | } |
421 | 20 | } |
422 | 10 | } |
423 | 1.17M | } else { |
424 | | // find index in vector _return_columns |
425 | | // for the read_orderby_key_num_prefix_columns orderby keys |
426 | 1.17M | for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { |
427 | 3.27k | for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { |
428 | 3.27k | if (_return_columns[idx] == i) { |
429 | 2.82k | _orderby_key_columns.push_back(idx); |
430 | 2.82k | break; |
431 | 2.82k | } |
432 | 3.27k | } |
433 | 2.81k | } |
434 | 1.17M | } |
435 | 1.18M | if (read_params.read_orderby_key_num_prefix_columns != _orderby_key_columns.size()) { |
436 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
437 | 0 | "read_orderby_key_num_prefix_columns != _orderby_key_columns.size, " |
438 | 0 | "read_params.read_orderby_key_num_prefix_columns={}, " |
439 | 0 | "_orderby_key_columns.size()={}", |
440 | 0 | read_params.read_orderby_key_num_prefix_columns, _orderby_key_columns.size()); |
441 | 0 | } |
442 | 1.18M | } |
443 | | |
444 | 1.22M | return Status::OK(); |
445 | 1.22M | } |
446 | | |
447 | 1.22M | Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { |
448 | 1.22M | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns); |
449 | 1.22M | std::vector<std::shared_ptr<ColumnPredicate>> predicates; |
450 | 1.22M | std::copy(read_params.predicates.cbegin(), read_params.predicates.cend(), |
451 | 1.22M | std::inserter(predicates, predicates.begin())); |
452 | | // Function filter push down to storage engine |
453 | 1.22M | auto is_like_predicate = [](std::shared_ptr<ColumnPredicate> _pred) { |
454 | 351 | return dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(_pred.get()) != nullptr || |
455 | 351 | dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(_pred.get()) != nullptr; |
456 | 351 | }; |
457 | | |
458 | 1.22M | for (const auto& filter : read_params.function_filters) { |
459 | 352 | predicates.emplace_back(_parse_to_predicate(filter)); |
460 | 352 | auto pred = predicates.back(); |
461 | | |
462 | 352 | const auto& col = _tablet_schema->column(pred->column_id()); |
463 | 352 | const auto* tablet_index = _tablet_schema->get_ngram_bf_index(col.unique_id()); |
464 | 352 | if (is_like_predicate(pred) && tablet_index && config::enable_query_like_bloom_filter) { |
465 | 16 | std::unique_ptr<segment_v2::BloomFilter> ng_bf; |
466 | 16 | std::string pattern = pred->get_search_str(); |
467 | 16 | auto gram_bf_size = tablet_index->get_gram_bf_size(); |
468 | 16 | auto gram_size = tablet_index->get_gram_size(); |
469 | | |
470 | 16 | RETURN_IF_ERROR(segment_v2::BloomFilter::create(segment_v2::NGRAM_BLOOM_FILTER, &ng_bf, |
471 | 16 | gram_bf_size)); |
472 | 16 | NgramTokenExtractor _token_extractor(gram_size); |
473 | | |
474 | 16 | if (_token_extractor.string_like_to_bloom_filter(pattern.data(), pattern.length(), |
475 | 16 | *ng_bf)) { |
476 | 16 | pred->set_page_ng_bf(std::move(ng_bf)); |
477 | 16 | } |
478 | 16 | } |
479 | 352 | } |
480 | | |
481 | 1.22M | int32_t delete_sign_idx = _tablet_schema->delete_sign_idx(); |
482 | 1.22M | for (auto predicate : predicates) { |
483 | 981k | auto column = _tablet_schema->column(predicate->column_id()); |
484 | 981k | if (column.aggregation() != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) { |
485 | | // When MOR value predicate pushdown is enabled, drop __DORIS_DELETE_SIGN__ |
486 | | // from storage-layer predicates entirely. Delete sign must only be evaluated |
487 | | // post-merge via VExpr to prevent deleted rows from reappearing. |
488 | 2.98k | if (read_params.enable_mor_value_predicate_pushdown && delete_sign_idx >= 0 && |
489 | 2.98k | predicate->column_id() == static_cast<uint32_t>(delete_sign_idx)) { |
490 | 24 | continue; |
491 | 24 | } |
492 | 2.95k | _value_col_predicates.push_back(predicate); |
493 | 978k | } else { |
494 | 978k | _col_predicates.push_back(predicate); |
495 | 978k | } |
496 | 981k | } |
497 | | |
498 | 1.22M | return Status::OK(); |
499 | 1.22M | } |
500 | | |
501 | | std::shared_ptr<ColumnPredicate> TabletReader::_parse_to_predicate( |
502 | 352 | const FunctionFilter& function_filter) { |
503 | 352 | int32_t index = _tablet_schema->field_index(function_filter._col_name); |
504 | 352 | if (index < 0) { |
505 | 0 | throw Exception(Status::InternalError("Column {} not found in tablet schema", |
506 | 0 | function_filter._col_name)); |
507 | 0 | return nullptr; |
508 | 0 | } |
509 | 352 | const TabletColumn& column = materialize_column(_tablet_schema->column(index)); |
510 | 352 | return create_column_predicate(index, std::make_shared<FunctionFilter>(function_filter), |
511 | 352 | column.type(), &column); |
512 | 352 | } |
513 | | |
514 | 1.22M | Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { |
515 | 1.22M | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns); |
516 | | // If it's cumu and not allow do delete when cumu |
517 | 1.22M | if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || |
518 | 1.22M | (read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION && |
519 | 1.22M | !config::enable_delete_when_cumu_compaction)) { |
520 | 25.0k | return Status::OK(); |
521 | 25.0k | } |
522 | 1.20M | bool cumu_delete = read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION && |
523 | 1.20M | config::enable_delete_when_cumu_compaction; |
524 | | // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates. |
525 | | // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. |
526 | | // `_delete_sign_available` indicates the condition where we could apply delete signs to data. |
527 | 1.20M | _delete_sign_available = (((read_params.reader_type == ReaderType::READER_BASE_COMPACTION || |
528 | 1.20M | read_params.reader_type == ReaderType::READER_FULL_COMPACTION) && |
529 | 1.20M | config::enable_prune_delete_sign_when_base_compaction) || |
530 | 1.20M | read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || |
531 | 1.20M | read_params.reader_type == ReaderType::READER_CHECKSUM); |
532 | | |
533 | | // `_filter_delete` indicates the condition where we should execlude deleted tuples when reading data. |
534 | | // However, queries will not use this condition but generate special where predicates to filter data. |
535 | | // (Though a lille bit confused, it is how the current logic working...) |
536 | 1.20M | _filter_delete = _delete_sign_available || cumu_delete; |
537 | 1.20M | return _delete_handler.init(_tablet_schema, read_params.delete_predicates, |
538 | 1.20M | read_params.version.second); |
539 | 1.22M | } |
540 | | |
541 | | Status TabletReader::init_reader_params_and_create_block( |
542 | | TabletSharedPtr tablet, ReaderType reader_type, |
543 | | const std::vector<RowsetSharedPtr>& input_rowsets, |
544 | 0 | TabletReader::ReaderParams* reader_params, Block* block) { |
545 | 0 | reader_params->tablet = tablet; |
546 | 0 | reader_params->reader_type = reader_type; |
547 | 0 | reader_params->version = |
548 | 0 | Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version()); |
549 | |
|
550 | 0 | TabletReadSource read_source; |
551 | 0 | for (const auto& rowset : input_rowsets) { |
552 | 0 | RowsetReaderSharedPtr rs_reader; |
553 | 0 | RETURN_IF_ERROR(rowset->create_reader(&rs_reader)); |
554 | 0 | read_source.rs_splits.emplace_back(std::move(rs_reader)); |
555 | 0 | } |
556 | 0 | read_source.fill_delete_predicates(); |
557 | 0 | reader_params->set_read_source(std::move(read_source)); |
558 | |
|
559 | 0 | std::vector<RowsetMetaSharedPtr> rowset_metas(input_rowsets.size()); |
560 | 0 | std::transform(input_rowsets.begin(), input_rowsets.end(), rowset_metas.begin(), |
561 | 0 | [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); }); |
562 | 0 | TabletSchemaSPtr read_tablet_schema = |
563 | 0 | tablet->tablet_schema_with_merged_max_schema_version(rowset_metas); |
564 | 0 | TabletSchemaSPtr merge_tablet_schema = std::make_shared<TabletSchema>(); |
565 | 0 | merge_tablet_schema->copy_from(*read_tablet_schema); |
566 | | |
567 | | // Merge the columns in delete predicate that not in latest schema in to current tablet schema |
568 | 0 | for (auto& del_pred : reader_params->delete_predicates) { |
569 | 0 | merge_tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); |
570 | 0 | } |
571 | 0 | reader_params->tablet_schema = merge_tablet_schema; |
572 | |
|
573 | 0 | reader_params->return_columns.resize(read_tablet_schema->num_columns()); |
574 | 0 | std::iota(reader_params->return_columns.begin(), reader_params->return_columns.end(), 0); |
575 | 0 | reader_params->origin_return_columns = &reader_params->return_columns; |
576 | |
|
577 | 0 | *block = read_tablet_schema->create_block(); |
578 | |
|
579 | 0 | return Status::OK(); |
580 | 0 | } |
581 | | |
582 | | } // namespace doris |