be/src/exprs/vcompound_pred.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | #include <gen_cpp/Opcodes_types.h> |
20 | | |
21 | | #include <algorithm> |
22 | | #include <cstdint> |
23 | | |
24 | | #include "common/logging.h" |
25 | | #include "common/status.h" |
26 | | #include "core/assert_cast.h" |
27 | | #include "core/column/column.h" |
28 | | #include "core/column/column_nullable.h" |
29 | | #include "exprs/vectorized_fn_call.h" |
30 | | #include "exprs/vexpr_context.h" |
31 | | #include "exprs/vexpr_fwd.h" |
32 | | #include "storage/index/zone_map/zonemap_eval_context.h" |
33 | | #include "util/simd/bits.h" |
34 | | |
35 | | namespace doris { |
36 | | |
37 | 48 | inline std::string compound_operator_to_string(TExprOpcode::type op) { |
38 | 48 | if (op == TExprOpcode::COMPOUND_AND) { |
39 | 25 | return "and"; |
40 | 25 | } else if (op == TExprOpcode::COMPOUND_OR) { |
41 | 21 | return "or"; |
42 | 21 | } else { |
43 | 2 | return "not"; |
44 | 2 | } |
45 | 48 | } |
46 | | |
47 | 3 | inline bool inverted_index_status_allows_row_fallback(const Status& status) { |
48 | 3 | DORIS_CHECK(!status.ok()); |
49 | 3 | return status.is<ErrorCode::INVERTED_INDEX_BYPASS>() || |
50 | 3 | status.is<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>() || |
51 | 3 | status.is<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>() || |
52 | 3 | status.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>() || |
53 | 3 | status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>(); |
54 | 3 | } |
55 | | |
56 | | class VCompoundPred : public VectorizedFnCall { |
57 | | ENABLE_FACTORY_CREATOR(VCompoundPred); |
58 | | |
59 | | public: |
60 | 48 | VCompoundPred(const TExprNode& node) : VectorizedFnCall(node) { |
61 | 48 | _op = node.opcode; |
62 | 48 | _fn.name.function_name = compound_operator_to_string(_op); |
63 | 48 | _expr_name = fmt::format("VCompoundPredicate[{}](arguments={},return={})", |
64 | 48 | _fn.name.function_name, get_child_names(), _data_type->get_name()); |
65 | 48 | } |
66 | | |
67 | | #ifdef BE_TEST |
68 | 2 | VCompoundPred() = default; |
69 | | #endif |
70 | | |
71 | 18 | const std::string& expr_name() const override { return _expr_name; } |
72 | 0 | Status clone_node(VExprSPtr* cloned_expr) const override { |
73 | 0 | DORIS_CHECK(cloned_expr != nullptr); |
74 | 0 | *cloned_expr = VCompoundPred::create_shared(clone_texpr_node()); |
75 | 0 | return Status::OK(); |
76 | 0 | } |
77 | | |
78 | | bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, |
79 | 24 | int column_id) const override { |
80 | 24 | return !_children.empty() && |
81 | 24 | (_op == TExprOpcode::COMPOUND_AND || _op == TExprOpcode::COMPOUND_OR) && |
82 | 27 | std::ranges::all_of(_children, [&](const VExprSPtr& child) { |
83 | 27 | return child->can_execute_on_raw_fixed_values(data_type, column_id); |
84 | 27 | }); |
85 | 24 | } |
86 | | |
87 | | Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, |
88 | | const DataTypePtr& data_type, int column_id, |
89 | 2 | uint8_t* matches) const override { |
90 | 2 | if (!can_execute_on_raw_fixed_values(data_type, column_id)) { |
91 | 0 | return Status::NotSupported("Compound predicate cannot evaluate raw fixed values"); |
92 | 0 | } |
93 | 2 | return _execute_raw_compound( |
94 | 4 | num_values, matches, [&](const VExprSPtr& child, uint8_t* child_matches) { |
95 | 4 | return child->execute_on_raw_fixed_values(values, num_values, value_width, |
96 | 4 | data_type, column_id, child_matches); |
97 | 4 | }); |
98 | 2 | } |
99 | | |
100 | | bool can_execute_on_raw_binary_values(const DataTypePtr& data_type, |
101 | 40 | int column_id) const override { |
102 | 40 | return !_children.empty() && |
103 | 40 | (_op == TExprOpcode::COMPOUND_AND || _op == TExprOpcode::COMPOUND_OR) && |
104 | 80 | std::ranges::all_of(_children, [&](const VExprSPtr& child) { |
105 | 80 | return child->can_execute_on_raw_binary_values(data_type, column_id); |
106 | 80 | }); |
107 | 40 | } |
108 | | |
109 | | Status execute_on_raw_binary_values(const StringRef* values, size_t num_values, |
110 | | const DataTypePtr& data_type, int column_id, |
111 | 8 | uint8_t* matches) const override { |
112 | 8 | if (!can_execute_on_raw_binary_values(data_type, column_id)) { |
113 | 0 | return Status::NotSupported("Compound predicate cannot evaluate raw binary values"); |
114 | 0 | } |
115 | 8 | return _execute_raw_compound( |
116 | 16 | num_values, matches, [&](const VExprSPtr& child, uint8_t* child_matches) { |
117 | 16 | return child->execute_on_raw_binary_values(values, num_values, data_type, |
118 | 16 | column_id, child_matches); |
119 | 16 | }); |
120 | 8 | } |
121 | | |
122 | 12 | bool raw_predicate_result_for_null() const override { |
123 | 12 | if (_op != TExprOpcode::COMPOUND_AND && _op != TExprOpcode::COMPOUND_OR) { |
124 | | // A Boolean keep bit cannot distinguish FALSE from UNKNOWN, so negating a child's |
125 | | // collapsed result is not SQL-correct. NOT remains residual and rejects NULL here. |
126 | 0 | return false; |
127 | 0 | } |
128 | 12 | if (_op == TExprOpcode::COMPOUND_AND) { |
129 | 3 | return std::ranges::all_of(_children, [](const VExprSPtr& child) { |
130 | 3 | return child->raw_predicate_result_for_null(); |
131 | 3 | }); |
132 | 3 | } |
133 | 18 | return std::ranges::any_of(_children, [](const VExprSPtr& child) { |
134 | 18 | return child->raw_predicate_result_for_null(); |
135 | 18 | }); |
136 | 12 | } |
137 | | |
138 | 62 | bool can_evaluate_zonemap_filter() const override { |
139 | 62 | switch (_op) { |
140 | 36 | case TExprOpcode::COMPOUND_AND: |
141 | 56 | return std::ranges::any_of(_children, [](const VExprSPtr& child) { |
142 | 56 | return child->can_evaluate_zonemap_filter(); |
143 | 56 | }); |
144 | 26 | case TExprOpcode::COMPOUND_OR: |
145 | 52 | return !_children.empty() && std::ranges::all_of(_children, [](const VExprSPtr& child) { |
146 | 52 | return child->can_evaluate_zonemap_filter(); |
147 | 52 | }); |
148 | 0 | case TExprOpcode::COMPOUND_NOT: |
149 | 0 | return false; |
150 | 0 | default: |
151 | 0 | return false; |
152 | 62 | } |
153 | 62 | } |
154 | | |
155 | 7 | ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { |
156 | 7 | switch (_op) { |
157 | 2 | case TExprOpcode::COMPOUND_AND: { |
158 | 4 | for (const auto& child : _children) { |
159 | 4 | if (!child->can_evaluate_zonemap_filter()) { |
160 | 1 | continue; |
161 | 1 | } |
162 | 3 | if (child->evaluate_zonemap_filter(ctx) == ZoneMapFilterResult::kNoMatch) { |
163 | 2 | return ZoneMapFilterResult::kNoMatch; |
164 | 2 | } |
165 | 3 | } |
166 | 0 | return ZoneMapFilterResult::kMayMatch; |
167 | 2 | } |
168 | 4 | case TExprOpcode::COMPOUND_OR: { |
169 | 6 | for (const auto& child : _children) { |
170 | 6 | DORIS_CHECK(child->can_evaluate_zonemap_filter()); |
171 | 6 | if (child->evaluate_zonemap_filter(ctx) != ZoneMapFilterResult::kNoMatch) { |
172 | 3 | return ZoneMapFilterResult::kMayMatch; |
173 | 3 | } |
174 | 6 | } |
175 | 1 | return ZoneMapFilterResult::kNoMatch; |
176 | 4 | } |
177 | 1 | case TExprOpcode::COMPOUND_NOT: |
178 | 1 | return unsupported_zonemap_filter(ctx); |
179 | 0 | default: |
180 | 0 | return unsupported_zonemap_filter(ctx); |
181 | 7 | } |
182 | 7 | } |
183 | | |
184 | 33 | bool can_evaluate_dictionary_filter() const override { |
185 | 33 | switch (_op) { |
186 | 30 | case TExprOpcode::COMPOUND_AND: |
187 | 33 | return std::ranges::any_of(_children, [](const VExprSPtr& child) { |
188 | 33 | return child->can_evaluate_dictionary_filter(); |
189 | 33 | }); |
190 | 3 | case TExprOpcode::COMPOUND_OR: |
191 | 6 | return !_children.empty() && std::ranges::all_of(_children, [](const VExprSPtr& child) { |
192 | 6 | return child->can_evaluate_dictionary_filter(); |
193 | 6 | }); |
194 | 0 | default: |
195 | 0 | return false; |
196 | 33 | } |
197 | 33 | } |
198 | | |
199 | 9 | bool is_safe_to_execute_on_selected_rows() const override { |
200 | | // Boolean composition introduces no data-dependent failure of its own. Reuse the generic |
201 | | // child walk so AND/OR remain eligible only when every nested expression is independently |
202 | | // safe; applying VectorizedFnCall's scalar-function allowlist to this structural node would |
203 | | // incorrectly disable selected-row execution for otherwise safe predicates. |
204 | 9 | return VExpr::is_safe_to_execute_on_selected_rows(); |
205 | 9 | } |
206 | | |
207 | | ZoneMapFilterResult evaluate_dictionary_filter( |
208 | 21 | const DictionaryEvalContext& ctx) const override { |
209 | 21 | switch (_op) { |
210 | 21 | case TExprOpcode::COMPOUND_AND: |
211 | 29 | for (const auto& child : _children) { |
212 | 29 | if (!child->can_evaluate_dictionary_filter()) { |
213 | 8 | continue; |
214 | 8 | } |
215 | 21 | if (child->evaluate_dictionary_filter(ctx) == ZoneMapFilterResult::kNoMatch) { |
216 | 13 | return ZoneMapFilterResult::kNoMatch; |
217 | 13 | } |
218 | 21 | } |
219 | 8 | return ZoneMapFilterResult::kMayMatch; |
220 | 0 | case TExprOpcode::COMPOUND_OR: |
221 | 0 | for (const auto& child : _children) { |
222 | 0 | DORIS_CHECK(child->can_evaluate_dictionary_filter()); |
223 | 0 | if (child->evaluate_dictionary_filter(ctx) != ZoneMapFilterResult::kNoMatch) { |
224 | 0 | return ZoneMapFilterResult::kMayMatch; |
225 | 0 | } |
226 | 0 | } |
227 | 0 | return ZoneMapFilterResult::kNoMatch; |
228 | 0 | default: |
229 | 0 | return ZoneMapFilterResult::kUnsupported; |
230 | 21 | } |
231 | 21 | } |
232 | | |
233 | 14 | bool can_evaluate_bloom_filter() const override { |
234 | 14 | switch (_op) { |
235 | 9 | case TExprOpcode::COMPOUND_AND: |
236 | 15 | return std::ranges::any_of(_children, [](const VExprSPtr& child) { |
237 | 15 | return child->can_evaluate_bloom_filter(); |
238 | 15 | }); |
239 | 5 | case TExprOpcode::COMPOUND_OR: |
240 | 8 | return !_children.empty() && std::ranges::all_of(_children, [](const VExprSPtr& child) { |
241 | 8 | return child->can_evaluate_bloom_filter(); |
242 | 8 | }); |
243 | 0 | default: |
244 | 0 | return false; |
245 | 14 | } |
246 | 14 | } |
247 | | |
248 | 0 | ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { |
249 | 0 | switch (_op) { |
250 | 0 | case TExprOpcode::COMPOUND_AND: |
251 | 0 | for (const auto& child : _children) { |
252 | 0 | if (!child->can_evaluate_bloom_filter()) { |
253 | 0 | continue; |
254 | 0 | } |
255 | 0 | if (child->evaluate_bloom_filter(ctx) == ZoneMapFilterResult::kNoMatch) { |
256 | 0 | return ZoneMapFilterResult::kNoMatch; |
257 | 0 | } |
258 | 0 | } |
259 | 0 | return ZoneMapFilterResult::kMayMatch; |
260 | 0 | case TExprOpcode::COMPOUND_OR: |
261 | 0 | for (const auto& child : _children) { |
262 | 0 | DORIS_CHECK(child->can_evaluate_bloom_filter()); |
263 | 0 | if (child->evaluate_bloom_filter(ctx) != ZoneMapFilterResult::kNoMatch) { |
264 | 0 | return ZoneMapFilterResult::kMayMatch; |
265 | 0 | } |
266 | 0 | } |
267 | 0 | return ZoneMapFilterResult::kNoMatch; |
268 | 0 | default: |
269 | 0 | return ZoneMapFilterResult::kUnsupported; |
270 | 0 | } |
271 | 0 | } |
272 | | |
273 | 3 | Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override { |
274 | 3 | segment_v2::InvertedIndexResultBitmap res; |
275 | 3 | bool all_pass = true; |
276 | | |
277 | 3 | switch (_op) { |
278 | 1 | case TExprOpcode::COMPOUND_OR: { |
279 | 1 | for (const auto& child : _children) { |
280 | 1 | if (Status st = child->evaluate_inverted_index(context, segment_num_rows); |
281 | 1 | !st.ok()) { |
282 | 1 | LOG(ERROR) << "expr:" << child->expr_name() |
283 | 1 | << " evaluate_inverted_index error:" << st.to_string(); |
284 | 1 | if (!inverted_index_status_allows_row_fallback(st)) { |
285 | 1 | return st; |
286 | 1 | } |
287 | 0 | all_pass = false; |
288 | 0 | continue; |
289 | 1 | } |
290 | 0 | auto inverted_index_context = context->get_index_context(); |
291 | 0 | if (inverted_index_context->has_index_result_for_expr(child.get())) { |
292 | 0 | const auto* index_result = |
293 | 0 | inverted_index_context->get_index_result_for_expr(child.get()); |
294 | 0 | if (res.is_empty()) { |
295 | 0 | res = *index_result; |
296 | 0 | } else { |
297 | 0 | res |= *index_result; |
298 | 0 | } |
299 | 0 | if (inverted_index_context->get_score_runtime() == nullptr) { |
300 | 0 | if (res.get_data_bitmap()->cardinality() == segment_num_rows) { |
301 | 0 | break; // Early exit if result is full |
302 | 0 | } |
303 | 0 | } |
304 | 0 | } else { |
305 | 0 | all_pass = false; |
306 | 0 | } |
307 | 0 | } |
308 | 0 | break; |
309 | 1 | } |
310 | 2 | case TExprOpcode::COMPOUND_AND: { |
311 | 2 | for (const auto& child : _children) { |
312 | 2 | if (Status st = child->evaluate_inverted_index(context, segment_num_rows); |
313 | 2 | !st.ok()) { |
314 | 2 | LOG(ERROR) << "expr:" << child->expr_name() |
315 | 2 | << " evaluate_inverted_index error:" << st.to_string(); |
316 | 2 | if (!inverted_index_status_allows_row_fallback(st)) { |
317 | 2 | return st; |
318 | 2 | } |
319 | 0 | all_pass = false; |
320 | 0 | continue; |
321 | 2 | } |
322 | 0 | if (context->get_index_context()->has_index_result_for_expr(child.get())) { |
323 | 0 | const auto* index_result = |
324 | 0 | context->get_index_context()->get_index_result_for_expr(child.get()); |
325 | 0 | if (res.is_empty()) { |
326 | 0 | res = *index_result; |
327 | 0 | } else { |
328 | 0 | res &= *index_result; |
329 | 0 | } |
330 | |
|
331 | 0 | if (res.get_data_bitmap()->isEmpty()) { |
332 | 0 | break; // Early exit if result is empty |
333 | 0 | } |
334 | 0 | } else { |
335 | 0 | all_pass = false; |
336 | 0 | } |
337 | 0 | } |
338 | 0 | break; |
339 | 2 | } |
340 | 0 | case TExprOpcode::COMPOUND_NOT: { |
341 | 0 | const auto& child = _children[0]; |
342 | 0 | Status st = child->evaluate_inverted_index(context, segment_num_rows); |
343 | 0 | if (!st.ok()) { |
344 | 0 | LOG(ERROR) << "expr:" << child->expr_name() |
345 | 0 | << " evaluate_inverted_index error:" << st.to_string(); |
346 | 0 | return st; |
347 | 0 | } |
348 | | |
349 | 0 | if (context->get_index_context()->has_index_result_for_expr(child.get())) { |
350 | 0 | const auto* index_result = |
351 | 0 | context->get_index_context()->get_index_result_for_expr(child.get()); |
352 | 0 | roaring::Roaring full_result; |
353 | 0 | full_result.addRange(0, segment_num_rows); |
354 | 0 | res = index_result->op_not(&full_result); |
355 | 0 | } else { |
356 | 0 | all_pass = false; |
357 | 0 | } |
358 | 0 | break; |
359 | 0 | } |
360 | 0 | default: |
361 | 0 | return Status::NotSupported( |
362 | 0 | "Compound operator must be AND, OR, or NOT to execute with inverted index."); |
363 | 3 | } |
364 | | |
365 | 0 | if (all_pass && !res.is_empty()) { |
366 | 0 | context->get_index_context()->set_index_result_for_expr(this, res); |
367 | 0 | } |
368 | 0 | return Status::OK(); |
369 | 3 | } |
370 | | |
371 | | Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, |
372 | 3 | size_t count, ColumnPtr& result_column) const override { |
373 | 3 | if (fast_execute(context, selector, count, result_column)) { |
374 | 0 | return Status::OK(); |
375 | 0 | } |
376 | 3 | if (get_num_children() == 1 || _has_const_child()) { |
377 | 0 | return VectorizedFnCall::execute_column_impl(context, block, selector, count, |
378 | 0 | result_column); |
379 | 0 | } |
380 | | |
381 | 3 | ColumnPtr lhs_column; |
382 | 3 | RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, lhs_column)); |
383 | 3 | lhs_column = lhs_column->convert_to_full_column_if_const(); |
384 | 3 | size_t size = lhs_column->size(); |
385 | | |
386 | 3 | bool lhs_is_nullable = lhs_column->is_nullable(); |
387 | 3 | auto [lhs_data_column, lhs_null_map] = |
388 | 3 | _get_raw_data_and_null_map(lhs_column, lhs_is_nullable); |
389 | 3 | size_t filted = simd::count_zero_num((int8_t*)lhs_data_column, size); |
390 | 3 | bool lhs_all_true = (filted == 0); |
391 | 3 | bool lhs_all_false = (filted == size); |
392 | | |
393 | 3 | bool lhs_all_is_not_null = false; |
394 | 3 | if (lhs_is_nullable) { |
395 | 3 | filted = simd::count_zero_num((int8_t*)lhs_null_map, size); |
396 | 3 | lhs_all_is_not_null = (filted == size); |
397 | 3 | } |
398 | | |
399 | 3 | ColumnPtr rhs_column = nullptr; |
400 | 3 | const uint8_t* __restrict rhs_data_column = nullptr; |
401 | 3 | const uint8_t* __restrict rhs_null_map = nullptr; |
402 | 3 | bool rhs_is_nullable = false; |
403 | 3 | bool rhs_all_true = false; |
404 | 3 | bool rhs_all_false = false; |
405 | 3 | bool rhs_all_is_not_null = false; |
406 | 3 | bool result_is_nullable = _data_type->is_nullable(); |
407 | | |
408 | 3 | auto get_rhs_colum = [&]() { |
409 | 3 | if (!rhs_column) { |
410 | 3 | RETURN_IF_ERROR( |
411 | 3 | _children[1]->execute_column(context, block, selector, count, rhs_column)); |
412 | 2 | rhs_column = rhs_column->convert_to_full_column_if_const(); |
413 | 2 | rhs_is_nullable = rhs_column->is_nullable(); |
414 | 2 | auto rhs_nullable_column = _get_raw_data_and_null_map(rhs_column, rhs_is_nullable); |
415 | 2 | rhs_data_column = rhs_nullable_column.first; |
416 | 2 | rhs_null_map = rhs_nullable_column.second; |
417 | 2 | size_t filted = simd::count_zero_num((int8_t*)rhs_data_column, size); |
418 | 2 | rhs_all_true = (filted == 0); |
419 | 2 | rhs_all_false = (filted == size); |
420 | 2 | if (rhs_is_nullable) { |
421 | 2 | filted = simd::count_zero_num((int8_t*)rhs_null_map, size); |
422 | 2 | rhs_all_is_not_null = (filted == size); |
423 | 2 | } |
424 | 2 | } |
425 | 2 | return Status::OK(); |
426 | 3 | }; |
427 | | |
428 | 3 | auto return_result_column_id = [&](ColumnPtr& arg_column) { |
429 | 2 | result_column = std::move(*arg_column).mutate(); |
430 | 2 | if (result_is_nullable && !result_column->is_nullable()) { |
431 | 0 | result_column = make_nullable(result_column); |
432 | 0 | } |
433 | 2 | }; |
434 | | |
435 | 3 | auto create_null_map_column = [&](ColumnPtr& null_map_column, |
436 | 3 | const uint8_t* __restrict null_map_data) { |
437 | 0 | if (null_map_data == nullptr) { |
438 | 0 | null_map_column = ColumnUInt8::create(size, 0); |
439 | 0 | null_map_data = |
440 | 0 | assert_cast<const ColumnUInt8*>(null_map_column.get())->get_data().data(); |
441 | 0 | } |
442 | 0 | return null_map_data; |
443 | 0 | }; |
444 | | |
445 | 3 | auto vector_vector = [&]<bool is_and_op>() { |
446 | 0 | MutableColumnPtr mutable_result_column; |
447 | 0 | uint8_t* __restrict result_data_column = nullptr; |
448 | 0 | const uint8_t* __restrict other_data_column = rhs_data_column; |
449 | 0 | if (lhs_column->use_count() == 1) { |
450 | 0 | mutable_result_column = IColumn::mutate(std::move(lhs_column)); |
451 | 0 | result_data_column = |
452 | 0 | assert_cast<ColumnUInt8*>(mutable_result_column.get())->get_data().data(); |
453 | 0 | } else if (rhs_column->use_count() == 1) { |
454 | 0 | mutable_result_column = IColumn::mutate(std::move(rhs_column)); |
455 | 0 | result_data_column = |
456 | 0 | assert_cast<ColumnUInt8*>(mutable_result_column.get())->get_data().data(); |
457 | 0 | other_data_column = lhs_data_column; |
458 | 0 | } else { |
459 | 0 | mutable_result_column = lhs_column->clone_resized(size); |
460 | 0 | result_data_column = |
461 | 0 | assert_cast<ColumnUInt8*>(mutable_result_column.get())->get_data().data(); |
462 | 0 | } |
463 | |
|
464 | 0 | do_not_null_pred<is_and_op>(result_data_column, other_data_column, size); |
465 | 0 | result_column = std::move(mutable_result_column); |
466 | 0 | }; Unexecuted instantiation: _ZZNK5doris13VCompoundPred19execute_column_implEPNS_12VExprContextEPKNS_5BlockEPKNS_8PODArrayIjLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEEmRNS_3COWINS_7IColumnEE13immutable_ptrISE_EEENKUlTnbvE_clILb1EEEDav Unexecuted instantiation: _ZZNK5doris13VCompoundPred19execute_column_implEPNS_12VExprContextEPKNS_5BlockEPKNS_8PODArrayIjLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEEmRNS_3COWINS_7IColumnEE13immutable_ptrISE_EEENKUlTnbvE_clILb0EEEDav |
467 | 3 | auto vector_vector_null = [&]<bool is_and_op>() { |
468 | 0 | auto col_res = ColumnUInt8::create(size); |
469 | 0 | auto col_nulls = ColumnUInt8::create(size); |
470 | |
|
471 | 0 | auto* __restrict res_datas = col_res->get_data().data(); |
472 | 0 | auto* __restrict res_nulls = col_nulls->get_data().data(); |
473 | 0 | ColumnPtr temp_null_map = nullptr; |
474 | | // maybe both children are nullable / or one of children is nullable |
475 | 0 | auto* __restrict lhs_null_map_tmp = create_null_map_column(temp_null_map, lhs_null_map); |
476 | 0 | auto* __restrict rhs_null_map_tmp = create_null_map_column(temp_null_map, rhs_null_map); |
477 | 0 | auto* __restrict lhs_data_column_tmp = lhs_data_column; |
478 | 0 | auto* __restrict rhs_data_column_tmp = rhs_data_column; |
479 | |
|
480 | 0 | do_null_pred<is_and_op>(lhs_data_column_tmp, lhs_null_map_tmp, rhs_data_column_tmp, |
481 | 0 | rhs_null_map_tmp, res_datas, res_nulls, size); |
482 | |
|
483 | 0 | result_column = ColumnNullable::create(std::move(col_res), std::move(col_nulls)); |
484 | 0 | }; Unexecuted instantiation: _ZZNK5doris13VCompoundPred19execute_column_implEPNS_12VExprContextEPKNS_5BlockEPKNS_8PODArrayIjLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEEmRNS_3COWINS_7IColumnEE13immutable_ptrISE_EEENKUlTnbvE0_clILb1EEEDav Unexecuted instantiation: _ZZNK5doris13VCompoundPred19execute_column_implEPNS_12VExprContextEPKNS_5BlockEPKNS_8PODArrayIjLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEEmRNS_3COWINS_7IColumnEE13immutable_ptrISE_EEENKUlTnbvE0_clILb0EEEDav |
485 | | |
486 | | // false and NULL ----> 0 |
487 | | // true and NULL ----> NULL |
488 | 3 | if (_op == TExprOpcode::COMPOUND_AND) { |
489 | | //1. not null column: all data is false |
490 | | //2. nullable column: null map all is not null |
491 | 1 | if ((lhs_all_false && !lhs_is_nullable) || (lhs_all_false && lhs_all_is_not_null)) { |
492 | | // false and any = false, return lhs |
493 | 0 | return_result_column_id(lhs_column); |
494 | 1 | } else { |
495 | 1 | RETURN_IF_ERROR(get_rhs_colum()); |
496 | | |
497 | 0 | if ((lhs_all_true && !lhs_is_nullable) || //not null column |
498 | 0 | (lhs_all_true && lhs_all_is_not_null)) { //nullable column |
499 | | // true and any = any, return rhs |
500 | |
|
501 | 0 | return_result_column_id(rhs_column); |
502 | 0 | } else if ((rhs_all_false && !rhs_is_nullable) || |
503 | 0 | (rhs_all_false && rhs_all_is_not_null)) { |
504 | | // any and false = false, return rhs |
505 | 0 | return_result_column_id(rhs_column); |
506 | 0 | } else if ((rhs_all_true && !rhs_is_nullable) || |
507 | 0 | (rhs_all_true && rhs_all_is_not_null)) { |
508 | | // any and true = any, return lhs |
509 | 0 | return_result_column_id(lhs_column); |
510 | 0 | } else { |
511 | 0 | if (!result_is_nullable) { |
512 | 0 | vector_vector.template operator()<true>(); |
513 | 0 | } else { |
514 | 0 | vector_vector_null.template operator()<true>(); |
515 | 0 | } |
516 | 0 | } |
517 | 0 | } |
518 | 2 | } else if (_op == TExprOpcode::COMPOUND_OR) { |
519 | | // true or NULL ----> 1 |
520 | | // false or NULL ----> NULL |
521 | 2 | if ((lhs_all_true && !lhs_is_nullable) || (lhs_all_true && lhs_all_is_not_null)) { |
522 | | // true or any = true, return lhs |
523 | 0 | return_result_column_id(lhs_column); |
524 | 2 | } else { |
525 | 2 | RETURN_IF_ERROR(get_rhs_colum()); |
526 | 2 | if ((lhs_all_false && !lhs_is_nullable) || (lhs_all_false && lhs_all_is_not_null)) { |
527 | | // false or any = any, return rhs |
528 | 1 | return_result_column_id(rhs_column); |
529 | 1 | } else if ((rhs_all_true && !rhs_is_nullable) || |
530 | 1 | (rhs_all_true && rhs_all_is_not_null)) { |
531 | | // any or true = true, return rhs |
532 | 0 | return_result_column_id(rhs_column); |
533 | 1 | } else if ((rhs_all_false && !rhs_is_nullable) || |
534 | 1 | (rhs_all_false && rhs_all_is_not_null)) { |
535 | | // any or false = any, return lhs |
536 | 1 | return_result_column_id(lhs_column); |
537 | 1 | } else { |
538 | 0 | if (!result_is_nullable) { |
539 | 0 | vector_vector.template operator()<false>(); |
540 | 0 | } else { |
541 | 0 | vector_vector_null.template operator()<false>(); |
542 | 0 | } |
543 | 0 | } |
544 | 2 | } |
545 | 2 | } else { |
546 | 0 | return Status::InternalError("Compound operator must be AND or OR."); |
547 | 0 | } |
548 | | |
549 | 3 | DCHECK_EQ(result_column->size(), count); |
550 | 2 | return Status::OK(); |
551 | 3 | } |
552 | | |
553 | 0 | double execute_cost() const override { |
554 | 0 | double cost = 0.3; |
555 | 0 | for (const auto& child : _children) { |
556 | 0 | cost += child->execute_cost(); |
557 | 0 | } |
558 | 0 | return cost; |
559 | 0 | } |
560 | | |
561 | | private: |
562 | | template <typename ExecuteChild> |
563 | | Status _execute_raw_compound(size_t num_values, uint8_t* matches, |
564 | 10 | ExecuteChild&& execute_child) const { |
565 | 10 | if (_op == TExprOpcode::COMPOUND_AND) { |
566 | 2 | for (const auto& child : _children) { |
567 | 2 | RETURN_IF_ERROR(execute_child(child, matches)); |
568 | 2 | } |
569 | 1 | return Status::OK(); |
570 | 1 | } |
571 | | |
572 | | // Each execution context owns its expression tree. Retaining masks on the OR node avoids |
573 | | // N+1 row-sized allocations for every decoder fragment, while nested OR nodes keep |
574 | | // independent buffers and therefore cannot overwrite their parent's in-flight state. |
575 | 9 | _raw_combined_scratch.resize(num_values); |
576 | 9 | std::ranges::fill(_raw_combined_scratch, 0); |
577 | 18 | for (const auto& child : _children) { |
578 | | // resize_fill() initializes only newly appended bytes; explicitly reset a reused mask |
579 | | // so matches from an earlier page fragment cannot leak into this OR evaluation. |
580 | 18 | _raw_child_scratch.resize(num_values); |
581 | 18 | std::ranges::fill(_raw_child_scratch, 1); |
582 | 18 | RETURN_IF_ERROR(execute_child(child, _raw_child_scratch.data())); |
583 | 154 | for (size_t row = 0; row < num_values; ++row) { |
584 | 136 | _raw_combined_scratch[row] |= _raw_child_scratch[row]; |
585 | 136 | } |
586 | 18 | } |
587 | | // Raw kernels receive an existing selection mask, so composition must preserve rows that |
588 | | // an earlier conjunct already rejected instead of replacing the caller's mask. |
589 | 77 | for (size_t row = 0; row < num_values; ++row) { |
590 | 68 | matches[row] &= _raw_combined_scratch[row]; |
591 | 68 | } |
592 | 9 | constexpr size_t MAX_RETAINED_RAW_MASK_BYTES = 1UL << 20; |
593 | 9 | if (_raw_combined_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { |
594 | 0 | IColumn::Filter().swap(_raw_combined_scratch); |
595 | 0 | } |
596 | 9 | if (_raw_child_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { |
597 | 0 | IColumn::Filter().swap(_raw_child_scratch); |
598 | 0 | } |
599 | 9 | return Status::OK(); |
600 | 9 | } _ZNK5doris13VCompoundPred21_execute_raw_compoundIZNKS0_27execute_on_raw_fixed_valuesEPKhmmRKSt10shared_ptrIKNS_9IDataTypeEEiPhEUlRKS4_INS_5VExprEESA_E_EENS_6StatusEmSA_OT_ Line | Count | Source | 564 | 2 | ExecuteChild&& execute_child) const { | 565 | 2 | if (_op == TExprOpcode::COMPOUND_AND) { | 566 | 2 | for (const auto& child : _children) { | 567 | 2 | RETURN_IF_ERROR(execute_child(child, matches)); | 568 | 2 | } | 569 | 1 | return Status::OK(); | 570 | 1 | } | 571 | | | 572 | | // Each execution context owns its expression tree. Retaining masks on the OR node avoids | 573 | | // N+1 row-sized allocations for every decoder fragment, while nested OR nodes keep | 574 | | // independent buffers and therefore cannot overwrite their parent's in-flight state. | 575 | 1 | _raw_combined_scratch.resize(num_values); | 576 | 1 | std::ranges::fill(_raw_combined_scratch, 0); | 577 | 2 | for (const auto& child : _children) { | 578 | | // resize_fill() initializes only newly appended bytes; explicitly reset a reused mask | 579 | | // so matches from an earlier page fragment cannot leak into this OR evaluation. | 580 | 2 | _raw_child_scratch.resize(num_values); | 581 | 2 | std::ranges::fill(_raw_child_scratch, 1); | 582 | 2 | RETURN_IF_ERROR(execute_child(child, _raw_child_scratch.data())); | 583 | 10 | for (size_t row = 0; row < num_values; ++row) { | 584 | 8 | _raw_combined_scratch[row] |= _raw_child_scratch[row]; | 585 | 8 | } | 586 | 2 | } | 587 | | // Raw kernels receive an existing selection mask, so composition must preserve rows that | 588 | | // an earlier conjunct already rejected instead of replacing the caller's mask. | 589 | 5 | for (size_t row = 0; row < num_values; ++row) { | 590 | 4 | matches[row] &= _raw_combined_scratch[row]; | 591 | 4 | } | 592 | 1 | constexpr size_t MAX_RETAINED_RAW_MASK_BYTES = 1UL << 20; | 593 | 1 | if (_raw_combined_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { | 594 | 0 | IColumn::Filter().swap(_raw_combined_scratch); | 595 | 0 | } | 596 | 1 | if (_raw_child_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { | 597 | 0 | IColumn::Filter().swap(_raw_child_scratch); | 598 | 0 | } | 599 | 1 | return Status::OK(); | 600 | 1 | } |
_ZNK5doris13VCompoundPred21_execute_raw_compoundIZNKS0_28execute_on_raw_binary_valuesEPKNS_9StringRefEmRKSt10shared_ptrIKNS_9IDataTypeEEiPhEUlRKS5_INS_5VExprEESB_E_EENS_6StatusEmSB_OT_ Line | Count | Source | 564 | 8 | ExecuteChild&& execute_child) const { | 565 | 8 | if (_op == TExprOpcode::COMPOUND_AND) { | 566 | 0 | for (const auto& child : _children) { | 567 | 0 | RETURN_IF_ERROR(execute_child(child, matches)); | 568 | 0 | } | 569 | 0 | return Status::OK(); | 570 | 0 | } | 571 | | | 572 | | // Each execution context owns its expression tree. Retaining masks on the OR node avoids | 573 | | // N+1 row-sized allocations for every decoder fragment, while nested OR nodes keep | 574 | | // independent buffers and therefore cannot overwrite their parent's in-flight state. | 575 | 8 | _raw_combined_scratch.resize(num_values); | 576 | 8 | std::ranges::fill(_raw_combined_scratch, 0); | 577 | 16 | for (const auto& child : _children) { | 578 | | // resize_fill() initializes only newly appended bytes; explicitly reset a reused mask | 579 | | // so matches from an earlier page fragment cannot leak into this OR evaluation. | 580 | 16 | _raw_child_scratch.resize(num_values); | 581 | 16 | std::ranges::fill(_raw_child_scratch, 1); | 582 | 16 | RETURN_IF_ERROR(execute_child(child, _raw_child_scratch.data())); | 583 | 144 | for (size_t row = 0; row < num_values; ++row) { | 584 | 128 | _raw_combined_scratch[row] |= _raw_child_scratch[row]; | 585 | 128 | } | 586 | 16 | } | 587 | | // Raw kernels receive an existing selection mask, so composition must preserve rows that | 588 | | // an earlier conjunct already rejected instead of replacing the caller's mask. | 589 | 72 | for (size_t row = 0; row < num_values; ++row) { | 590 | 64 | matches[row] &= _raw_combined_scratch[row]; | 591 | 64 | } | 592 | 8 | constexpr size_t MAX_RETAINED_RAW_MASK_BYTES = 1UL << 20; | 593 | 8 | if (_raw_combined_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { | 594 | 0 | IColumn::Filter().swap(_raw_combined_scratch); | 595 | 0 | } | 596 | 8 | if (_raw_child_scratch.capacity() > MAX_RETAINED_RAW_MASK_BYTES) { | 597 | 0 | IColumn::Filter().swap(_raw_child_scratch); | 598 | 0 | } | 599 | 8 | return Status::OK(); | 600 | 8 | } |
|
601 | | |
602 | | mutable IColumn::Filter _raw_combined_scratch; |
603 | | mutable IColumn::Filter _raw_child_scratch; |
604 | | |
605 | 0 | static inline constexpr uint8_t apply_and_null(UInt8 a, UInt8 l_null, UInt8 b, UInt8 r_null) { |
606 | | // (<> && false) is false, (true && NULL) is NULL |
607 | 0 | return (l_null & r_null) | (r_null & (l_null ^ a)) | (l_null & (r_null ^ b)); |
608 | 0 | } |
609 | 0 | static inline constexpr uint8_t apply_or_null(UInt8 a, UInt8 l_null, UInt8 b, UInt8 r_null) { |
610 | | // (<> || true) is true, (false || NULL) is NULL |
611 | 0 | return (l_null & r_null) | (r_null & (r_null ^ a)) | (l_null & (l_null ^ b)); |
612 | 0 | } |
613 | | |
614 | | template <bool is_and> |
615 | | void static do_not_null_pred(uint8_t* __restrict lhs, const uint8_t* __restrict rhs, |
616 | 0 | size_t size) { |
617 | | #ifdef NDEBUG |
618 | | #if defined(__clang__) |
619 | | #pragma clang loop vectorize(enable) |
620 | | #elif defined(__GNUC__) && (__GNUC__ >= 5) |
621 | | #pragma GCC ivdep |
622 | | #endif |
623 | | #endif |
624 | 0 | for (size_t i = 0; i < size; ++i) { |
625 | 0 | if constexpr (is_and) { |
626 | 0 | lhs[i] &= rhs[i]; |
627 | 0 | } else { |
628 | 0 | lhs[i] |= rhs[i]; |
629 | 0 | } |
630 | 0 | } |
631 | 0 | } Unexecuted instantiation: _ZN5doris13VCompoundPred16do_not_null_predILb1EEEvPhPKhm Unexecuted instantiation: _ZN5doris13VCompoundPred16do_not_null_predILb0EEEvPhPKhm |
632 | | |
633 | | template <bool is_and> |
634 | | void static do_null_pred(const uint8_t* __restrict lhs_data, const uint8_t* __restrict lhs_null, |
635 | | const uint8_t* __restrict rhs_data, const uint8_t* __restrict rhs_null, |
636 | | uint8_t* __restrict res_data, uint8_t* __restrict res_null, |
637 | 0 | size_t size) { |
638 | | #ifdef NDEBUG |
639 | | #if defined(__clang__) |
640 | | #pragma clang loop vectorize(enable) |
641 | | #elif defined(__GNUC__) && (__GNUC__ >= 5) |
642 | | #pragma GCC ivdep |
643 | | #endif |
644 | | #endif |
645 | 0 | for (size_t i = 0; i < size; ++i) { |
646 | 0 | if constexpr (is_and) { |
647 | 0 | res_null[i] = apply_and_null(lhs_data[i], lhs_null[i], rhs_data[i], rhs_null[i]); |
648 | 0 | res_data[i] = lhs_data[i] & rhs_data[i]; |
649 | 0 | } else { |
650 | 0 | res_null[i] = apply_or_null(lhs_data[i], lhs_null[i], rhs_data[i], rhs_null[i]); |
651 | 0 | res_data[i] = lhs_data[i] | rhs_data[i]; |
652 | 0 | } |
653 | 0 | } |
654 | 0 | } Unexecuted instantiation: _ZN5doris13VCompoundPred12do_null_predILb1EEEvPKhS3_S3_S3_PhS4_m Unexecuted instantiation: _ZN5doris13VCompoundPred12do_null_predILb0EEEvPKhS3_S3_S3_PhS4_m |
655 | | |
656 | 3 | bool _has_const_child() const { |
657 | 3 | return std::ranges::any_of(_children, |
658 | 6 | [](const VExprSPtr& arg) -> bool { return arg->is_constant(); }); |
659 | 3 | } |
660 | | |
661 | | std::pair<const uint8_t*, const uint8_t*> _get_raw_data_and_null_map( |
662 | 5 | const ColumnPtr& column, bool has_nullable_column) const { |
663 | 5 | if (has_nullable_column) { |
664 | 5 | const auto* nullable_column = assert_cast<const ColumnNullable*>(column.get()); |
665 | 5 | auto* data_column = |
666 | 5 | assert_cast<const ColumnUInt8*>(nullable_column->get_nested_column_ptr().get()) |
667 | 5 | ->get_data() |
668 | 5 | .data(); |
669 | 5 | auto* null_map = nullable_column->get_null_map_column_ptr()->get_data().data(); |
670 | 5 | return std::make_pair(data_column, null_map); |
671 | 5 | } else { |
672 | 0 | auto* data_column = assert_cast<const ColumnUInt8*>(column.get())->get_data().data(); |
673 | 0 | return std::make_pair(data_column, nullptr); |
674 | 0 | } |
675 | 5 | } |
676 | | |
677 | | TExprOpcode::type _op; |
678 | | }; |
679 | | |
680 | | } // namespace doris |