/root/doris/be/src/olap/tablet_reader.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "olap/tablet_reader.h" |
19 | | |
20 | | #include <gen_cpp/olap_file.pb.h> |
21 | | #include <gen_cpp/segment_v2.pb.h> |
22 | | #include <thrift/protocol/TDebugProtocol.h> |
23 | | |
24 | | #include <algorithm> |
25 | | #include <functional> |
26 | | #include <iterator> |
27 | | #include <memory> |
28 | | #include <numeric> |
29 | | #include <ostream> |
30 | | #include <shared_mutex> |
31 | | |
32 | | #include "common/compiler_util.h" // IWYU pragma: keep |
33 | | #include "common/config.h" |
34 | | #include "common/exception.h" |
35 | | #include "common/logging.h" |
36 | | #include "common/status.h" |
37 | | #include "exprs/bitmapfilter_predicate.h" |
38 | | #include "exprs/bloom_filter_func.h" |
39 | | #include "exprs/create_predicate_function.h" |
40 | | #include "exprs/hybrid_set.h" |
41 | | #include "olap/column_predicate.h" |
42 | | #include "olap/itoken_extractor.h" |
43 | | #include "olap/like_column_predicate.h" |
44 | | #include "olap/olap_common.h" |
45 | | #include "olap/olap_define.h" |
46 | | #include "olap/predicate_creator.h" |
47 | | #include "olap/row_cursor.h" |
48 | | #include "olap/rowset/segment_v2/bloom_filter.h" |
49 | | #include "olap/schema.h" |
50 | | #include "olap/tablet.h" |
51 | | #include "olap/tablet_meta.h" |
52 | | #include "olap/tablet_schema.h" |
53 | | #include "runtime/query_context.h" |
54 | | #include "runtime/runtime_predicate.h" |
55 | | #include "runtime/runtime_state.h" |
56 | | #include "vec/common/arena.h" |
57 | | #include "vec/common/variant_util.h" |
58 | | #include "vec/core/block.h" |
59 | | |
60 | | namespace doris { |
61 | | #include "common/compile_check_begin.h" |
62 | | using namespace ErrorCode; |
63 | | |
64 | 361 | void TabletReader::ReaderParams::check_validation() const { |
65 | 361 | if (UNLIKELY(version.first == -1 && is_segcompaction == false)) { |
66 | 0 | throw Exception(Status::FatalError("version is not set. tablet={}", tablet->tablet_id())); |
67 | 0 | } |
68 | 361 | } |
69 | | |
70 | 0 | std::string TabletReader::ReaderParams::to_string() const { |
71 | 0 | std::stringstream ss; |
72 | 0 | ss << "tablet=" << tablet->tablet_id() << " reader_type=" << int(reader_type) |
73 | 0 | << " aggregation=" << aggregation << " version=" << version |
74 | 0 | << " start_key_include=" << start_key_include << " end_key_include=" << end_key_include; |
75 | |
|
76 | 0 | for (const auto& key : start_key) { |
77 | 0 | ss << " keys=" << key; |
78 | 0 | } |
79 | |
|
80 | 0 | for (const auto& key : end_key) { |
81 | 0 | ss << " end_keys=" << key; |
82 | 0 | } |
83 | |
|
84 | 0 | return ss.str(); |
85 | 0 | } |
86 | | |
87 | 361 | Status TabletReader::init(const ReaderParams& read_params) { |
88 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); |
89 | | |
90 | 361 | Status res = _init_params(read_params); |
91 | 361 | if (!res.ok()) { |
92 | 0 | LOG(WARNING) << "fail to init reader when init params. res:" << res |
93 | 0 | << ", tablet_id:" << read_params.tablet->tablet_id() |
94 | 0 | << ", schema_hash:" << read_params.tablet->schema_hash() |
95 | 0 | << ", reader type:" << int(read_params.reader_type) |
96 | 0 | << ", version:" << read_params.version; |
97 | 0 | } |
98 | 361 | return res; |
99 | 361 | } |
100 | | |
101 | 339 | Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { |
102 | 339 | SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns); |
103 | 339 | if (read_params.rs_splits.empty()) { |
104 | 0 | return Status::InternalError("fail to acquire data sources. tablet={}", |
105 | 0 | _tablet->tablet_id()); |
106 | 0 | } |
107 | | |
108 | 339 | bool eof = false; |
109 | 339 | bool is_lower_key_included = _keys_param.start_key_include; |
110 | 339 | bool is_upper_key_included = _keys_param.end_key_include; |
111 | | |
112 | 339 | for (int i = 0; i < _keys_param.start_keys.size(); ++i) { |
113 | | // lower bound |
114 | 0 | RowCursor& start_key = _keys_param.start_keys[i]; |
115 | 0 | RowCursor& end_key = _keys_param.end_keys[i]; |
116 | |
|
117 | 0 | if (!is_lower_key_included) { |
118 | 0 | if (compare_row_key(start_key, end_key) >= 0) { |
119 | 0 | VLOG_NOTICE << "return EOF when lower key not include" |
120 | 0 | << ", start_key=" << start_key.to_string() |
121 | 0 | << ", end_key=" << end_key.to_string(); |
122 | 0 | eof = true; |
123 | 0 | break; |
124 | 0 | } |
125 | 0 | } else { |
126 | 0 | if (compare_row_key(start_key, end_key) > 0) { |
127 | 0 | VLOG_NOTICE << "return EOF when lower key include=" |
128 | 0 | << ", start_key=" << start_key.to_string() |
129 | 0 | << ", end_key=" << end_key.to_string(); |
130 | 0 | eof = true; |
131 | 0 | break; |
132 | 0 | } |
133 | 0 | } |
134 | | |
135 | 0 | _is_lower_keys_included.push_back(is_lower_key_included); |
136 | 0 | _is_upper_keys_included.push_back(is_upper_key_included); |
137 | 0 | } |
138 | | |
139 | 339 | if (eof) { |
140 | 0 | return Status::EndOfFile("reach end of scan range. tablet={}", _tablet->tablet_id()); |
141 | 0 | } |
142 | | |
143 | 339 | bool need_ordered_result = true; |
144 | 339 | if (read_params.reader_type == ReaderType::READER_QUERY) { |
145 | 0 | if (_tablet_schema->keys_type() == DUP_KEYS) { |
146 | | // duplicated keys are allowed, no need to merge sort keys in rowset |
147 | 0 | need_ordered_result = false; |
148 | 0 | } |
149 | 0 | if (_tablet_schema->keys_type() == UNIQUE_KEYS && |
150 | 0 | _tablet->enable_unique_key_merge_on_write()) { |
151 | | // unique keys with merge on write, no need to merge sort keys in rowset |
152 | 0 | need_ordered_result = false; |
153 | 0 | } |
154 | 0 | if (_aggregation) { |
155 | | // compute engine will aggregate rows with the same key, |
156 | | // it's ok for rowset to return unordered result |
157 | 0 | need_ordered_result = false; |
158 | 0 | } |
159 | |
|
160 | 0 | if (_direct_mode) { |
161 | | // direct mode indicates that the storage layer does not need to merge, |
162 | | // it's ok for rowset to return unordered result |
163 | 0 | need_ordered_result = false; |
164 | 0 | } |
165 | |
|
166 | 0 | if (read_params.read_orderby_key) { |
167 | 0 | need_ordered_result = true; |
168 | 0 | } |
169 | 0 | } |
170 | | |
171 | 339 | _reader_context.reader_type = read_params.reader_type; |
172 | 339 | _reader_context.version = read_params.version; |
173 | 339 | _reader_context.tablet_schema = _tablet_schema; |
174 | 339 | _reader_context.need_ordered_result = need_ordered_result; |
175 | 339 | _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids; |
176 | 339 | _reader_context.topn_filter_target_node_id = read_params.topn_filter_target_node_id; |
177 | 339 | _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; |
178 | 339 | _reader_context.read_orderby_key_limit = read_params.read_orderby_key_limit; |
179 | 339 | _reader_context.filter_block_conjuncts = read_params.filter_block_conjuncts; |
180 | 339 | _reader_context.return_columns = &_return_columns; |
181 | 339 | _reader_context.read_orderby_key_columns = |
182 | 339 | !_orderby_key_columns.empty() ? &_orderby_key_columns : nullptr; |
183 | 339 | _reader_context.predicates = &_col_predicates; |
184 | 339 | _reader_context.value_predicates = &_value_col_predicates; |
185 | 339 | _reader_context.lower_bound_keys = &_keys_param.start_keys; |
186 | 339 | _reader_context.is_lower_keys_included = &_is_lower_keys_included; |
187 | 339 | _reader_context.upper_bound_keys = &_keys_param.end_keys; |
188 | 339 | _reader_context.is_upper_keys_included = &_is_upper_keys_included; |
189 | 339 | _reader_context.delete_handler = &_delete_handler; |
190 | 339 | _reader_context.stats = &_stats; |
191 | 339 | _reader_context.use_page_cache = read_params.use_page_cache; |
192 | 339 | _reader_context.sequence_id_idx = _sequence_col_idx; |
193 | 339 | _reader_context.is_unique = tablet()->keys_type() == UNIQUE_KEYS; |
194 | 339 | _reader_context.merged_rows = &_merged_rows; |
195 | 339 | _reader_context.delete_bitmap = read_params.delete_bitmap; |
196 | 339 | _reader_context.enable_unique_key_merge_on_write = tablet()->enable_unique_key_merge_on_write(); |
197 | 339 | _reader_context.record_rowids = read_params.record_rowids; |
198 | 339 | _reader_context.rowid_conversion = read_params.rowid_conversion; |
199 | 339 | _reader_context.is_key_column_group = read_params.is_key_column_group; |
200 | 339 | _reader_context.remaining_conjunct_roots = read_params.remaining_conjunct_roots; |
201 | 339 | _reader_context.common_expr_ctxs_push_down = read_params.common_expr_ctxs_push_down; |
202 | 339 | _reader_context.output_columns = &read_params.output_columns; |
203 | 339 | _reader_context.push_down_agg_type_opt = read_params.push_down_agg_type_opt; |
204 | 339 | _reader_context.ttl_seconds = _tablet->ttl_seconds(); |
205 | 339 | _reader_context.score_runtime = read_params.score_runtime; |
206 | 339 | _reader_context.collection_statistics = read_params.collection_statistics; |
207 | | |
208 | 339 | _reader_context.virtual_column_exprs = read_params.virtual_column_exprs; |
209 | 339 | _reader_context.vir_cid_to_idx_in_block = read_params.vir_cid_to_idx_in_block; |
210 | 339 | _reader_context.vir_col_idx_to_type = read_params.vir_col_idx_to_type; |
211 | 339 | _reader_context.ann_topn_runtime = read_params.ann_topn_runtime; |
212 | | |
213 | 339 | _reader_context.condition_cache_digest = read_params.condition_cache_digest; |
214 | 339 | _reader_context.all_access_paths = read_params.all_access_paths; |
215 | 339 | _reader_context.predicate_access_paths = read_params.predicate_access_paths; |
216 | | |
217 | 339 | return Status::OK(); |
218 | 339 | } |
219 | | |
220 | 0 | TabletColumn TabletReader::materialize_column(const TabletColumn& orig) { |
221 | 0 | if (!orig.is_variant_type()) { |
222 | 0 | return orig; |
223 | 0 | } |
224 | 0 | TabletColumn column_with_cast_type = orig; |
225 | 0 | auto cast_type = _reader_context.target_cast_type_for_variants.at(orig.name()); |
226 | 0 | return vectorized::variant_util::get_column_by_type( |
227 | 0 | cast_type, orig.name(), |
228 | 0 | { |
229 | 0 | .unique_id = orig.unique_id(), |
230 | 0 | .parent_unique_id = orig.parent_unique_id(), |
231 | 0 | .path_info = *orig.path_info_ptr(), |
232 | 0 | }); |
233 | 0 | } |
234 | | |
235 | 361 | Status TabletReader::_init_params(const ReaderParams& read_params) { |
236 | 361 | read_params.check_validation(); |
237 | | |
238 | 361 | _direct_mode = read_params.direct_mode; |
239 | 361 | _aggregation = read_params.aggregation; |
240 | 361 | _reader_type = read_params.reader_type; |
241 | 361 | _tablet = read_params.tablet; |
242 | 361 | _tablet_schema = read_params.tablet_schema; |
243 | 361 | _reader_context.runtime_state = read_params.runtime_state; |
244 | 361 | _reader_context.target_cast_type_for_variants = read_params.target_cast_type_for_variants; |
245 | | |
246 | 361 | RETURN_IF_ERROR(_init_conditions_param(read_params)); |
247 | | |
248 | 361 | Status res = _init_delete_condition(read_params); |
249 | 361 | if (!res.ok()) { |
250 | 0 | LOG(WARNING) << "fail to init delete param. res = " << res; |
251 | 0 | return res; |
252 | 0 | } |
253 | | |
254 | 361 | res = _init_return_columns(read_params); |
255 | 361 | if (!res.ok()) { |
256 | 0 | LOG(WARNING) << "fail to init return columns. res = " << res; |
257 | 0 | return res; |
258 | 0 | } |
259 | | |
260 | 361 | res = _init_keys_param(read_params); |
261 | 361 | if (!res.ok()) { |
262 | 0 | LOG(WARNING) << "fail to init keys param. res=" << res; |
263 | 0 | return res; |
264 | 0 | } |
265 | 361 | res = _init_orderby_keys_param(read_params); |
266 | 361 | if (!res.ok()) { |
267 | 0 | LOG(WARNING) << "fail to init orderby keys param. res=" << res; |
268 | 0 | return res; |
269 | 0 | } |
270 | 361 | if (_tablet_schema->has_sequence_col()) { |
271 | 8 | auto sequence_col_idx = _tablet_schema->sequence_col_idx(); |
272 | 8 | DCHECK_NE(sequence_col_idx, -1); |
273 | 16 | for (auto col : _return_columns) { |
274 | | // query has sequence col |
275 | 16 | if (col == sequence_col_idx) { |
276 | 4 | _sequence_col_idx = sequence_col_idx; |
277 | 4 | break; |
278 | 4 | } |
279 | 16 | } |
280 | 8 | } |
281 | | |
282 | 361 | return res; |
283 | 361 | } |
284 | | |
285 | 361 | Status TabletReader::_init_return_columns(const ReaderParams& read_params) { |
286 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns); |
287 | 361 | if (read_params.reader_type == ReaderType::READER_QUERY) { |
288 | 22 | _return_columns = read_params.return_columns; |
289 | 22 | _tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set; |
290 | 37 | for (auto id : read_params.return_columns) { |
291 | 37 | if (_tablet_schema->column(id).is_key()) { |
292 | 22 | _key_cids.push_back(id); |
293 | 22 | } else { |
294 | 15 | _value_cids.push_back(id); |
295 | 15 | } |
296 | 37 | } |
297 | 339 | } else if (read_params.return_columns.empty()) { |
298 | 0 | for (uint32_t i = 0; i < _tablet_schema->num_columns(); ++i) { |
299 | 0 | _return_columns.push_back(i); |
300 | 0 | if (_tablet_schema->column(i).is_key()) { |
301 | 0 | _key_cids.push_back(i); |
302 | 0 | } else { |
303 | 0 | _value_cids.push_back(i); |
304 | 0 | } |
305 | 0 | } |
306 | 0 | VLOG_NOTICE << "return column is empty, using full column as default."; |
307 | 339 | } else if ((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION || |
308 | 339 | read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || |
309 | 339 | read_params.reader_type == ReaderType::READER_BASE_COMPACTION || |
310 | 339 | read_params.reader_type == ReaderType::READER_FULL_COMPACTION || |
311 | 339 | read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || |
312 | 339 | read_params.reader_type == ReaderType::READER_ALTER_TABLE) && |
313 | 339 | !read_params.return_columns.empty()) { |
314 | 339 | _return_columns = read_params.return_columns; |
315 | 1.07k | for (auto id : read_params.return_columns) { |
316 | 1.07k | if (_tablet_schema->column(id).is_key()) { |
317 | 115 | _key_cids.push_back(id); |
318 | 961 | } else { |
319 | 961 | _value_cids.push_back(id); |
320 | 961 | } |
321 | 1.07k | } |
322 | 339 | } else if (read_params.reader_type == ReaderType::READER_CHECKSUM) { |
323 | 0 | _return_columns = read_params.return_columns; |
324 | 0 | for (auto id : read_params.return_columns) { |
325 | 0 | if (_tablet_schema->column(id).is_key()) { |
326 | 0 | _key_cids.push_back(id); |
327 | 0 | } else { |
328 | 0 | _value_cids.push_back(id); |
329 | 0 | } |
330 | 0 | } |
331 | 0 | } else { |
332 | 0 | return Status::Error<INVALID_ARGUMENT>( |
333 | 0 | "fail to init return columns. reader_type={}, return_columns_size={}", |
334 | 0 | int(read_params.reader_type), read_params.return_columns.size()); |
335 | 0 | } |
336 | | |
337 | 361 | std::sort(_key_cids.begin(), _key_cids.end(), std::greater<>()); |
338 | | |
339 | 361 | return Status::OK(); |
340 | 361 | } |
341 | | |
342 | 361 | Status TabletReader::_init_keys_param(const ReaderParams& read_params) { |
343 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns); |
344 | 361 | if (read_params.start_key.empty()) { |
345 | 361 | return Status::OK(); |
346 | 361 | } |
347 | | |
348 | 0 | _keys_param.start_key_include = read_params.start_key_include; |
349 | 0 | _keys_param.end_key_include = read_params.end_key_include; |
350 | |
|
351 | 0 | size_t start_key_size = read_params.start_key.size(); |
352 | | //_keys_param.start_keys.resize(start_key_size); |
353 | 0 | std::vector<RowCursor>(start_key_size).swap(_keys_param.start_keys); |
354 | |
|
355 | 0 | size_t scan_key_size = read_params.start_key.front().size(); |
356 | 0 | if (scan_key_size > _tablet_schema->num_columns()) { |
357 | 0 | return Status::Error<INVALID_ARGUMENT>( |
358 | 0 | "Input param are invalid. Column count is bigger than num_columns of schema. " |
359 | 0 | "column_count={}, schema.num_columns={}", |
360 | 0 | scan_key_size, _tablet_schema->num_columns()); |
361 | 0 | } |
362 | | |
363 | 0 | std::vector<uint32_t> columns(scan_key_size); |
364 | 0 | std::iota(columns.begin(), columns.end(), 0); |
365 | |
|
366 | 0 | std::shared_ptr<Schema> schema = std::make_shared<Schema>(_tablet_schema->columns(), columns); |
367 | |
|
368 | 0 | for (size_t i = 0; i < start_key_size; ++i) { |
369 | 0 | if (read_params.start_key[i].size() != scan_key_size) { |
370 | 0 | return Status::Error<INVALID_ARGUMENT>( |
371 | 0 | "The start_key.at({}).size={}, not equals the scan_key_size={}", i, |
372 | 0 | read_params.start_key[i].size(), scan_key_size); |
373 | 0 | } |
374 | | |
375 | 0 | Status res = _keys_param.start_keys[i].init_scan_key( |
376 | 0 | _tablet_schema, read_params.start_key[i].values(), schema); |
377 | 0 | if (!res.ok()) { |
378 | 0 | LOG(WARNING) << "fail to init row cursor. res = " << res; |
379 | 0 | return res; |
380 | 0 | } |
381 | 0 | res = _keys_param.start_keys[i].from_tuple(read_params.start_key[i]); |
382 | 0 | if (!res.ok()) { |
383 | 0 | LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << "key_index=" << i; |
384 | 0 | return res; |
385 | 0 | } |
386 | 0 | } |
387 | | |
388 | 0 | size_t end_key_size = read_params.end_key.size(); |
389 | | //_keys_param.end_keys.resize(end_key_size); |
390 | 0 | std::vector<RowCursor>(end_key_size).swap(_keys_param.end_keys); |
391 | 0 | for (size_t i = 0; i < end_key_size; ++i) { |
392 | 0 | if (read_params.end_key[i].size() != scan_key_size) { |
393 | 0 | return Status::Error<INVALID_ARGUMENT>( |
394 | 0 | "The end_key.at({}).size={}, not equals the scan_key_size={}", i, |
395 | 0 | read_params.end_key[i].size(), scan_key_size); |
396 | 0 | } |
397 | | |
398 | 0 | Status res = _keys_param.end_keys[i].init_scan_key(_tablet_schema, |
399 | 0 | read_params.end_key[i].values(), schema); |
400 | 0 | if (!res.ok()) { |
401 | 0 | LOG(WARNING) << "fail to init row cursor. res = " << res; |
402 | 0 | return res; |
403 | 0 | } |
404 | | |
405 | 0 | res = _keys_param.end_keys[i].from_tuple(read_params.end_key[i]); |
406 | 0 | if (!res.ok()) { |
407 | 0 | LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << " key_index=" << i; |
408 | 0 | return res; |
409 | 0 | } |
410 | 0 | } |
411 | | |
412 | | //TODO:check the valid of start_key and end_key.(eg. start_key <= end_key) |
413 | | |
414 | 0 | return Status::OK(); |
415 | 0 | } |
416 | | |
417 | 361 | Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) { |
418 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns); |
419 | | // UNIQUE_KEYS will compare all keys as before |
420 | 361 | if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS && |
421 | 299 | _tablet->enable_unique_key_merge_on_write())) { |
422 | 299 | if (!_tablet_schema->cluster_key_uids().empty()) { |
423 | 0 | if (read_params.read_orderby_key_num_prefix_columns > |
424 | 0 | _tablet_schema->cluster_key_uids().size()) { |
425 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
426 | 0 | "read_orderby_key_num_prefix_columns={} > cluster_keys.size()={}", |
427 | 0 | read_params.read_orderby_key_num_prefix_columns, |
428 | 0 | _tablet_schema->cluster_key_uids().size()); |
429 | 0 | } |
430 | 0 | for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { |
431 | 0 | auto cid = _tablet_schema->cluster_key_uids()[i]; |
432 | 0 | auto index = _tablet_schema->field_index(cid); |
433 | 0 | if (index < 0) { |
434 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
435 | 0 | "could not find cluster key column with unique_id=" + |
436 | 0 | std::to_string(cid) + |
437 | 0 | " in tablet schema, tablet_id=" + std::to_string(_tablet->tablet_id())); |
438 | 0 | } |
439 | 0 | for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { |
440 | 0 | if (_return_columns[idx] == index) { |
441 | 0 | _orderby_key_columns.push_back(idx); |
442 | 0 | break; |
443 | 0 | } |
444 | 0 | } |
445 | 0 | } |
446 | 299 | } else { |
447 | | // find index in vector _return_columns |
448 | | // for the read_orderby_key_num_prefix_columns orderby keys |
449 | 299 | for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) { |
450 | 0 | for (uint32_t idx = 0; idx < _return_columns.size(); idx++) { |
451 | 0 | if (_return_columns[idx] == i) { |
452 | 0 | _orderby_key_columns.push_back(idx); |
453 | 0 | break; |
454 | 0 | } |
455 | 0 | } |
456 | 0 | } |
457 | 299 | } |
458 | 299 | if (read_params.read_orderby_key_num_prefix_columns != _orderby_key_columns.size()) { |
459 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
460 | 0 | "read_orderby_key_num_prefix_columns != _orderby_key_columns.size, " |
461 | 0 | "read_params.read_orderby_key_num_prefix_columns={}, " |
462 | 0 | "_orderby_key_columns.size()={}", |
463 | 0 | read_params.read_orderby_key_num_prefix_columns, _orderby_key_columns.size()); |
464 | 0 | } |
465 | 299 | } |
466 | | |
467 | 361 | return Status::OK(); |
468 | 361 | } |
469 | | |
470 | 361 | Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { |
471 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns); |
472 | 361 | std::vector<std::shared_ptr<ColumnPredicate>> predicates; |
473 | 361 | std::copy(read_params.predicates.cbegin(), read_params.predicates.cend(), |
474 | 361 | std::inserter(predicates, predicates.begin())); |
475 | | // Function filter push down to storage engine |
476 | 361 | auto is_like_predicate = [](std::shared_ptr<ColumnPredicate> _pred) { |
477 | 0 | return dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(_pred.get()) != nullptr || |
478 | 0 | dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(_pred.get()) != nullptr; |
479 | 0 | }; |
480 | | |
481 | 361 | for (const auto& filter : read_params.function_filters) { |
482 | 0 | predicates.emplace_back(_parse_to_predicate(filter)); |
483 | 0 | auto pred = predicates.back(); |
484 | |
|
485 | 0 | const auto& col = _tablet_schema->column(pred->column_id()); |
486 | 0 | const auto* tablet_index = _tablet_schema->get_ngram_bf_index(col.unique_id()); |
487 | 0 | if (is_like_predicate(pred) && tablet_index && config::enable_query_like_bloom_filter) { |
488 | 0 | std::unique_ptr<segment_v2::BloomFilter> ng_bf; |
489 | 0 | std::string pattern = pred->get_search_str(); |
490 | 0 | auto gram_bf_size = tablet_index->get_gram_bf_size(); |
491 | 0 | auto gram_size = tablet_index->get_gram_size(); |
492 | |
|
493 | 0 | RETURN_IF_ERROR(segment_v2::BloomFilter::create(segment_v2::NGRAM_BLOOM_FILTER, &ng_bf, |
494 | 0 | gram_bf_size)); |
495 | 0 | NgramTokenExtractor _token_extractor(gram_size); |
496 | |
|
497 | 0 | if (_token_extractor.string_like_to_bloom_filter(pattern.data(), pattern.length(), |
498 | 0 | *ng_bf)) { |
499 | 0 | pred->set_page_ng_bf(std::move(ng_bf)); |
500 | 0 | } |
501 | 0 | } |
502 | 0 | } |
503 | | |
504 | 361 | for (auto predicate : predicates) { |
505 | 0 | auto column = _tablet_schema->column(predicate->column_id()); |
506 | 0 | if (column.aggregation() != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) { |
507 | 0 | _value_col_predicates.push_back(predicate); |
508 | 0 | } else { |
509 | 0 | _col_predicates.push_back(predicate); |
510 | 0 | } |
511 | 0 | } |
512 | | |
513 | 361 | return Status::OK(); |
514 | 361 | } |
515 | | |
516 | | std::shared_ptr<ColumnPredicate> TabletReader::_parse_to_predicate( |
517 | 0 | const FunctionFilter& function_filter) { |
518 | 0 | int32_t index = _tablet_schema->field_index(function_filter._col_name); |
519 | 0 | if (index < 0) { |
520 | 0 | throw Exception(Status::InternalError("Column {} not found in tablet schema", |
521 | 0 | function_filter._col_name)); |
522 | 0 | return nullptr; |
523 | 0 | } |
524 | 0 | const TabletColumn& column = materialize_column(_tablet_schema->column(index)); |
525 | 0 | return create_column_predicate(index, std::make_shared<FunctionFilter>(function_filter), |
526 | 0 | column.type(), &column); |
527 | 0 | } |
528 | | |
529 | 361 | Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { |
530 | 361 | SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns); |
531 | | // If it's cumu and not allow do delete when cumu |
532 | 361 | if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || |
533 | 361 | (read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION && |
534 | 361 | !config::enable_delete_when_cumu_compaction)) { |
535 | 0 | return Status::OK(); |
536 | 0 | } |
537 | 361 | bool cumu_delete = read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION && |
538 | 361 | config::enable_delete_when_cumu_compaction; |
539 | | // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates. |
540 | | // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. |
541 | | // `_delete_sign_available` indicates the condition where we could apply delete signs to data. |
542 | 361 | _delete_sign_available = (((read_params.reader_type == ReaderType::READER_BASE_COMPACTION || |
543 | 361 | read_params.reader_type == ReaderType::READER_FULL_COMPACTION) && |
544 | 361 | config::enable_prune_delete_sign_when_base_compaction) || |
545 | 361 | read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || |
546 | 361 | read_params.reader_type == ReaderType::READER_CHECKSUM); |
547 | | |
548 | | // `_filter_delete` indicates the condition where we should execlude deleted tuples when reading data. |
549 | | // However, queries will not use this condition but generate special where predicates to filter data. |
550 | | // (Though a lille bit confused, it is how the current logic working...) |
551 | 361 | _filter_delete = _delete_sign_available || cumu_delete; |
552 | 361 | return _delete_handler.init(_tablet_schema, read_params.delete_predicates, |
553 | 361 | read_params.version.second); |
554 | 361 | } |
555 | | |
556 | | Status TabletReader::init_reader_params_and_create_block( |
557 | | TabletSharedPtr tablet, ReaderType reader_type, |
558 | | const std::vector<RowsetSharedPtr>& input_rowsets, |
559 | 0 | TabletReader::ReaderParams* reader_params, vectorized::Block* block) { |
560 | 0 | reader_params->tablet = tablet; |
561 | 0 | reader_params->reader_type = reader_type; |
562 | 0 | reader_params->version = |
563 | 0 | Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version()); |
564 | |
|
565 | 0 | TabletReadSource read_source; |
566 | 0 | for (const auto& rowset : input_rowsets) { |
567 | 0 | RowsetReaderSharedPtr rs_reader; |
568 | 0 | RETURN_IF_ERROR(rowset->create_reader(&rs_reader)); |
569 | 0 | read_source.rs_splits.emplace_back(std::move(rs_reader)); |
570 | 0 | } |
571 | 0 | read_source.fill_delete_predicates(); |
572 | 0 | reader_params->set_read_source(std::move(read_source)); |
573 | |
|
574 | 0 | std::vector<RowsetMetaSharedPtr> rowset_metas(input_rowsets.size()); |
575 | 0 | std::transform(input_rowsets.begin(), input_rowsets.end(), rowset_metas.begin(), |
576 | 0 | [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); }); |
577 | 0 | TabletSchemaSPtr read_tablet_schema = |
578 | 0 | tablet->tablet_schema_with_merged_max_schema_version(rowset_metas); |
579 | 0 | TabletSchemaSPtr merge_tablet_schema = std::make_shared<TabletSchema>(); |
580 | 0 | merge_tablet_schema->copy_from(*read_tablet_schema); |
581 | | |
582 | | // Merge the columns in delete predicate that not in latest schema in to current tablet schema |
583 | 0 | for (auto& del_pred : reader_params->delete_predicates) { |
584 | 0 | merge_tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); |
585 | 0 | } |
586 | 0 | reader_params->tablet_schema = merge_tablet_schema; |
587 | |
|
588 | 0 | reader_params->return_columns.resize(read_tablet_schema->num_columns()); |
589 | 0 | std::iota(reader_params->return_columns.begin(), reader_params->return_columns.end(), 0); |
590 | 0 | reader_params->origin_return_columns = &reader_params->return_columns; |
591 | |
|
592 | 0 | *block = read_tablet_schema->create_block(); |
593 | |
|
594 | 0 | return Status::OK(); |
595 | 0 | } |
596 | | |
597 | | #include "common/compile_check_end.h" |
598 | | } // namespace doris |