be/src/exprs/vbloom_predicate.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "exprs/vbloom_predicate.h" |
19 | | |
20 | | #include <cstddef> |
21 | | #include <utility> |
22 | | |
23 | | #include "common/status.h" |
24 | | #include "core/block/block.h" |
25 | | #include "core/block/column_numbers.h" |
26 | | #include "core/block/column_with_type_and_name.h" |
27 | | #include "core/column/column.h" |
28 | | #include "core/column/column_nullable.h" |
29 | | #include "core/column/column_vector.h" |
30 | | #include "core/data_type/data_type.h" |
31 | | #include "core/data_type/data_type_nullable.h" |
32 | | #include "core/types.h" |
33 | | #include "exprs/bloom_filter_func.h" |
34 | | #include "exprs/expr_zonemap_filter.h" |
35 | | #include "exprs/vslot_ref.h" |
36 | | #include "runtime/runtime_state.h" |
37 | | |
38 | | namespace doris { |
39 | | class RowDescriptor; |
40 | | class TExprNode; |
41 | | |
42 | | } // namespace doris |
43 | | |
44 | | namespace doris { |
45 | | |
46 | | class VExprContext; |
47 | | |
48 | 5.38k | VBloomPredicate::VBloomPredicate(const TExprNode& node) : VExpr(node), _filter(nullptr) {} |
49 | | |
50 | | Status VBloomPredicate::prepare(RuntimeState* state, const RowDescriptor& desc, |
51 | 5.33k | VExprContext* context) { |
52 | 5.33k | RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); |
53 | | |
54 | 5.33k | if (_children.size() != 1) { |
55 | 0 | return Status::InternalError("Invalid argument for VBloomPredicate."); |
56 | 0 | } |
57 | | |
58 | 5.33k | _prepare_finished = true; |
59 | 5.33k | return Status::OK(); |
60 | 5.33k | } |
61 | | |
62 | | Status VBloomPredicate::open(RuntimeState* state, VExprContext* context, |
63 | 6.41k | FunctionContext::FunctionStateScope scope) { |
64 | 6.41k | DCHECK(_prepare_finished); |
65 | 6.41k | RETURN_IF_ERROR(VExpr::open(state, context, scope)); |
66 | 6.41k | _open_finished = true; |
67 | 6.41k | return Status::OK(); |
68 | 6.41k | } |
69 | | |
70 | 6.57k | void VBloomPredicate::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { |
71 | 6.57k | VExpr::close(context, scope); |
72 | 6.57k | } |
73 | | |
74 | | Status VBloomPredicate::_do_execute(VExprContext* context, const Block* block, |
75 | | const uint8_t* __restrict filter, const Selector* selector, |
76 | 1.54k | size_t count, ColumnPtr& result_column) const { |
77 | 1.54k | DCHECK(_open_finished || block == nullptr); |
78 | 1.54k | DCHECK(!(filter != nullptr && selector != nullptr)) |
79 | 0 | << "filter and selector can not be both set"; |
80 | 1.54k | DCHECK_EQ(_children.size(), 1); |
81 | | |
82 | 1.54k | ColumnPtr argument_column; |
83 | 1.54k | RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, argument_column)); |
84 | 1.54k | argument_column = argument_column->convert_to_full_column_if_const(); |
85 | | |
86 | 1.54k | size_t sz = argument_column->size(); |
87 | 1.54k | auto res_data_column = ColumnUInt8::create(sz); |
88 | | |
89 | 1.54k | res_data_column->resize(sz); |
90 | 1.54k | auto* ptr = ((ColumnUInt8*)res_data_column.get())->get_data().data(); |
91 | | |
92 | 1.54k | _filter->find_fixed_len(argument_column, ptr, filter); |
93 | | |
94 | 1.54k | result_column = std::move(res_data_column); |
95 | 1.54k | DCHECK_EQ(result_column->size(), count); |
96 | 1.54k | return Status::OK(); |
97 | 1.54k | } |
98 | | |
99 | | Status VBloomPredicate::execute_column_impl(VExprContext* context, const Block* block, |
100 | | const Selector* selector, size_t count, |
101 | 257 | ColumnPtr& result_column) const { |
102 | 257 | return _do_execute(context, block, nullptr, selector, count, result_column); |
103 | 257 | } |
104 | | |
105 | | Status VBloomPredicate::execute_runtime_filter(VExprContext* context, const Block* block, |
106 | | const uint8_t* __restrict filter, size_t count, |
107 | | ColumnPtr& result_column, |
108 | 1.28k | ColumnPtr* arg_column) const { |
109 | 1.28k | return _do_execute(context, block, filter, nullptr, count, result_column); |
110 | 1.28k | } |
111 | | |
112 | | namespace { |
113 | | |
114 | 120 | bool bloom_filter_type_matches(PrimitiveType filter_type, const DataTypePtr& data_type) { |
115 | 120 | if (data_type == nullptr) { |
116 | 0 | return false; |
117 | 0 | } |
118 | 120 | const auto value_type = remove_nullable(data_type)->get_primitive_type(); |
119 | 120 | return filter_type == value_type || (is_string_type(filter_type) && is_string_type(value_type)); |
120 | 120 | } |
121 | | |
122 | | } // namespace |
123 | | |
124 | | bool VBloomPredicate::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, |
125 | 38 | int column_id) const { |
126 | 38 | if (_filter == nullptr || !_filter->supports_raw_fixed_values() || _children.size() != 1) { |
127 | 5 | return false; |
128 | 5 | } |
129 | 33 | const auto slot = std::dynamic_pointer_cast<VSlotRef>(_children[0]); |
130 | 33 | return slot != nullptr && slot->column_id() == column_id && |
131 | 33 | bloom_filter_type_matches(_filter->primitive_type(), slot->data_type()) && |
132 | 33 | bloom_filter_type_matches(_filter->primitive_type(), data_type); |
133 | 38 | } |
134 | | |
135 | | Status VBloomPredicate::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, |
136 | | size_t value_width, |
137 | | const DataTypePtr& data_type, int column_id, |
138 | 3 | uint8_t* matches) const { |
139 | 3 | if (!can_execute_on_raw_fixed_values(data_type, column_id)) { |
140 | 0 | return Status::NotSupported("Bloom predicate cannot evaluate raw fixed-width values"); |
141 | 0 | } |
142 | | // Hash physical values inside BloomFilterFunc<T>; reconstructing an untyped hash here could |
143 | | // disagree with the build-side hash for dates, decimals, and other fixed-width wrappers. |
144 | 3 | return _filter->find_batch_raw_fixed(values, num_values, value_width, matches); |
145 | 3 | } |
146 | | |
147 | | bool VBloomPredicate::can_execute_on_raw_binary_values(const DataTypePtr& data_type, |
148 | 16 | int column_id) const { |
149 | 16 | if (_filter == nullptr || !_filter->supports_raw_binary_values() || _children.size() != 1) { |
150 | 1 | return false; |
151 | 1 | } |
152 | 15 | const auto slot = std::dynamic_pointer_cast<VSlotRef>(_children[0]); |
153 | 15 | return slot != nullptr && slot->column_id() == column_id && |
154 | 15 | bloom_filter_type_matches(_filter->primitive_type(), slot->data_type()) && |
155 | 15 | bloom_filter_type_matches(_filter->primitive_type(), data_type); |
156 | 16 | } |
157 | | |
158 | | Status VBloomPredicate::execute_on_raw_binary_values(const StringRef* values, size_t num_values, |
159 | | const DataTypePtr& data_type, int column_id, |
160 | 2 | uint8_t* matches) const { |
161 | 2 | if (!can_execute_on_raw_binary_values(data_type, column_id)) { |
162 | 0 | return Status::NotSupported("Bloom predicate cannot evaluate raw binary values"); |
163 | 0 | } |
164 | 2 | return _filter->find_batch_raw_binary(values, num_values, matches); |
165 | 2 | } |
166 | | |
167 | | ZoneMapFilterResult VBloomPredicate::evaluate_dictionary_filter( |
168 | 3 | const DictionaryEvalContext& ctx) const { |
169 | 3 | if (!can_evaluate_dictionary_filter()) { |
170 | 0 | return ZoneMapFilterResult::kUnsupported; |
171 | 0 | } |
172 | 3 | const auto slot = std::dynamic_pointer_cast<VSlotRef>(_children[0]); |
173 | 3 | DORIS_CHECK(slot != nullptr); |
174 | 3 | const auto* dictionary = ctx.slot(slot->column_id()); |
175 | 3 | if (dictionary == nullptr || |
176 | 3 | !bloom_filter_type_matches(_filter->primitive_type(), dictionary->data_type)) { |
177 | 0 | return ZoneMapFilterResult::kUnsupported; |
178 | 0 | } |
179 | 4 | for (const auto& value : dictionary->values) { |
180 | 4 | if (_filter->test_field(value)) { |
181 | 2 | return ZoneMapFilterResult::kMayMatch; |
182 | 2 | } |
183 | 4 | } |
184 | 1 | return ZoneMapFilterResult::kNoMatch; |
185 | 3 | } |
186 | | |
187 | 21 | bool VBloomPredicate::can_evaluate_dictionary_filter() const { |
188 | 21 | if (_filter == nullptr || _children.size() != 1) { |
189 | 0 | return false; |
190 | 0 | } |
191 | 21 | const auto slot = std::dynamic_pointer_cast<VSlotRef>(_children[0]); |
192 | 21 | return slot != nullptr && |
193 | 21 | bloom_filter_type_matches(_filter->primitive_type(), slot->data_type()); |
194 | 21 | } |
195 | | |
196 | 5.30k | const std::string& VBloomPredicate::expr_name() const { |
197 | 5.30k | return EXPR_NAME; |
198 | 5.30k | } |
199 | | |
200 | 5.39k | void VBloomPredicate::set_filter(std::shared_ptr<BloomFilterFuncBase> filter) { |
201 | 5.39k | _filter = filter; |
202 | 5.39k | } |
203 | | |
204 | 4.88k | uint64_t VBloomPredicate::get_digest(uint64_t seed) const { |
205 | 4.88k | seed = _children[0]->get_digest(seed); |
206 | 4.90k | if (seed) { |
207 | 4.90k | char* data; |
208 | 4.90k | int len; |
209 | 4.90k | _filter->get_data(&data, &len); |
210 | 4.90k | return HashUtil::hash64(data, len, seed); |
211 | 4.90k | } |
212 | 18.4E | return 0; |
213 | 4.88k | } |
214 | | |
215 | | } // namespace doris |