/root/doris/be/src/exprs/function/function.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/IFunction.cpp |
19 | | // and modified by Doris |
20 | | |
21 | | #include "exprs/function/function.h" |
22 | | |
23 | | #include <algorithm> |
24 | | #include <memory> |
25 | | #include <numeric> |
26 | | |
27 | | #include "common/status.h" |
28 | | #include "core/assert_cast.h" |
29 | | #include "core/column/column.h" |
30 | | #include "core/column/column_const.h" |
31 | | #include "core/column/column_nullable.h" |
32 | | #include "core/column/column_vector.h" |
33 | | #include "core/data_type/data_type_array.h" |
34 | | #include "core/data_type/data_type_nothing.h" |
35 | | #include "core/data_type/data_type_nullable.h" |
36 | | #include "core/data_type/define_primitive_type.h" |
37 | | #include "core/data_type/primitive_type.h" |
38 | | #include "core/field.h" |
39 | | #include "exec/common/util.hpp" |
40 | | #include "exprs/aggregate/aggregate_function.h" |
41 | | #include "exprs/function/function_helpers.h" |
42 | | #include "storage/index/zone_map/zonemap_eval_context.h" |
43 | | |
44 | | namespace doris { |
45 | | #include "common/compile_check_begin.h" |
46 | | ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const ColumnNumbers& args, |
47 | 70.8k | size_t input_rows_count) { |
48 | 70.8k | ColumnPtr result_null_map_column; |
49 | | /// If result is already nullable. |
50 | 70.8k | ColumnPtr src_not_nullable = src; |
51 | 70.8k | MutableColumnPtr mutable_result_null_map_column; |
52 | | |
53 | 70.8k | if (const auto* nullable = check_and_get_column<ColumnNullable>(*src)) { |
54 | 8.66k | src_not_nullable = nullable->get_nested_column_ptr(); |
55 | 8.66k | result_null_map_column = nullable->get_null_map_column_ptr(); |
56 | 8.66k | } |
57 | | |
58 | 97.4k | for (const auto& arg : args) { |
59 | 97.4k | const ColumnWithTypeAndName& elem = block.get_by_position(arg); |
60 | 97.4k | if (!elem.type->is_nullable() || is_column_const(*elem.column)) { |
61 | 21.1k | continue; |
62 | 21.1k | } |
63 | | |
64 | 76.3k | if (const auto* nullable = assert_cast<const ColumnNullable*>(elem.column.get()); |
65 | 76.3k | nullable->has_null()) { |
66 | 1.06k | const ColumnPtr& null_map_column = nullable->get_null_map_column_ptr(); |
67 | 1.06k | if (!result_null_map_column) { // NOLINT(bugprone-use-after-move) |
68 | 953 | result_null_map_column = null_map_column->clone_resized(input_rows_count); |
69 | 953 | continue; |
70 | 953 | } |
71 | | |
72 | 115 | if (!mutable_result_null_map_column) { |
73 | 90 | mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); |
74 | 90 | } |
75 | | |
76 | 115 | NullMap& result_null_map = |
77 | 115 | assert_cast<ColumnUInt8&>(*mutable_result_null_map_column).get_data(); |
78 | 115 | const NullMap& src_null_map = |
79 | 115 | assert_cast<const ColumnUInt8&>(*null_map_column).get_data(); |
80 | | |
81 | 115 | VectorizedUtils::update_null_map(result_null_map, src_null_map); |
82 | 115 | } |
83 | 76.3k | } |
84 | | |
85 | | // Commit merged null map back: result_null_map_column was moved into |
86 | | // mutable_result_null_map_column when merging 2+ nullable args with nulls. |
87 | 70.8k | if (mutable_result_null_map_column) { |
88 | 90 | result_null_map_column = std::move(mutable_result_null_map_column); |
89 | 90 | } |
90 | | |
91 | 70.8k | if (!result_null_map_column) { |
92 | 61.2k | if (is_column_const(*src)) { |
93 | 2 | return ColumnConst::create( |
94 | 2 | make_nullable(assert_cast<const ColumnConst&>(*src).get_data_column_ptr(), |
95 | 2 | false), |
96 | 2 | input_rows_count); |
97 | 2 | } |
98 | 61.2k | return ColumnNullable::create(src, ColumnUInt8::create(input_rows_count, 0)); |
99 | 61.2k | } |
100 | | |
101 | 9.62k | return ColumnNullable::create(src_not_nullable, result_null_map_column); |
102 | 70.8k | } |
103 | | |
104 | 18.6k | bool have_null_column(const Block& block, const ColumnNumbers& args) { |
105 | 33.9k | return std::ranges::any_of(args, [&block](const auto& elem) { |
106 | 33.9k | return block.get_by_position(elem).type->is_nullable(); |
107 | 33.9k | }); |
108 | 18.6k | } |
109 | | |
110 | 13.6k | bool have_null_column(const ColumnsWithTypeAndName& args) { |
111 | 13.6k | return std::ranges::any_of(args, [](const auto& elem) { return elem.type->is_nullable(); }); |
112 | 13.6k | } |
113 | | |
114 | | inline Status PreparedFunctionImpl::_execute_skipped_constant_deal(FunctionContext* context, |
115 | | Block& block, |
116 | | const ColumnNumbers& args, |
117 | | uint32_t result, |
118 | 107k | size_t input_rows_count) const { |
119 | 107k | bool executed = false; |
120 | 107k | RETURN_IF_ERROR(default_implementation_for_nulls(context, block, args, result, input_rows_count, |
121 | 107k | &executed)); |
122 | 107k | if (executed) { |
123 | 13.4k | return Status::OK(); |
124 | 13.4k | } |
125 | 94.3k | return execute_impl(context, block, args, result, input_rows_count); |
126 | 107k | } |
127 | | |
128 | | Status PreparedFunctionImpl::default_implementation_for_constant_arguments( |
129 | | FunctionContext* context, Block& block, const ColumnNumbers& args, uint32_t result, |
130 | 107k | size_t input_rows_count, bool* executed) const { |
131 | 107k | *executed = false; |
132 | 107k | ColumnNumbers args_expect_const = get_arguments_that_are_always_constant(); |
133 | | |
134 | | // Check that these arguments are really constant. |
135 | 107k | for (auto arg_num : args_expect_const) { |
136 | 84.0k | if (arg_num < args.size() && |
137 | 84.0k | !is_column_const(*block.get_by_position(args[arg_num]).column)) { |
138 | 0 | return Status::InvalidArgument("Argument at index {} for function {} must be constant", |
139 | 0 | arg_num, get_name()); |
140 | 0 | } |
141 | 84.0k | } |
142 | | |
143 | 107k | if (args.empty() || !use_default_implementation_for_constants() || |
144 | 107k | !VectorizedUtils::all_arguments_are_constant(block, args)) { |
145 | 104k | return Status::OK(); |
146 | 104k | } |
147 | | |
148 | | // now all columns are const. |
149 | 2.88k | Block temporary_block; |
150 | | |
151 | 2.88k | int arguments_size = (int)args.size(); |
152 | 9.53k | for (size_t arg_num = 0; arg_num < arguments_size; ++arg_num) { |
153 | 6.65k | const ColumnWithTypeAndName& column = block.get_by_position(args[arg_num]); |
154 | | // Columns in const_list --> column_const, others --> nested_column |
155 | | // that's because some functions supposes some specific columns always constant. |
156 | | // If we unpack it, there will be unnecessary cost of virtual judge. |
157 | 6.65k | if (args_expect_const.end() != |
158 | 6.65k | std::find(args_expect_const.begin(), args_expect_const.end(), arg_num)) { |
159 | 0 | temporary_block.simple_insert({column.column, column.type, column.name}); |
160 | 6.65k | } else { |
161 | 6.65k | temporary_block.simple_insert( |
162 | 6.65k | {assert_cast<const ColumnConst*>(column.column.get())->get_data_column_ptr(), |
163 | 6.65k | column.type, column.name}); |
164 | 6.65k | } |
165 | 6.65k | } |
166 | | |
167 | 2.88k | temporary_block.simple_insert(block.get_by_position(result)); |
168 | | |
169 | 2.88k | ColumnNumbers temporary_argument_numbers(arguments_size); |
170 | 9.53k | for (int i = 0; i < arguments_size; ++i) { |
171 | 6.65k | temporary_argument_numbers[i] = i; |
172 | 6.65k | } |
173 | | |
174 | 2.88k | RETURN_IF_ERROR(_execute_skipped_constant_deal(context, temporary_block, |
175 | 2.88k | temporary_argument_numbers, arguments_size, |
176 | 2.88k | temporary_block.rows())); |
177 | | |
178 | 2.88k | ColumnPtr result_column; |
179 | | /// extremely rare case, when we have function with completely const arguments |
180 | | /// but some of them produced by non is_deterministic function |
181 | 2.88k | if (temporary_block.get_by_position(arguments_size).column->size() > 1) { |
182 | 0 | result_column = temporary_block.get_by_position(arguments_size).column->clone_resized(1); |
183 | 2.88k | } else { |
184 | 2.88k | result_column = temporary_block.get_by_position(arguments_size).column; |
185 | 2.88k | } |
186 | | // We shuold handle the case where the result column is also a ColumnConst. |
187 | 2.88k | block.get_by_position(result).column = ColumnConst::create(result_column, input_rows_count); |
188 | 2.88k | *executed = true; |
189 | 2.88k | return Status::OK(); |
190 | 2.88k | } |
191 | | |
192 | | Status PreparedFunctionImpl::default_implementation_for_nulls( |
193 | | FunctionContext* context, Block& block, const ColumnNumbers& args, uint32_t result, |
194 | 107k | size_t input_rows_count, bool* executed) const { |
195 | 107k | *executed = false; |
196 | 107k | if (args.empty() || !use_default_implementation_for_nulls()) { |
197 | 84.9k | return Status::OK(); |
198 | 84.9k | } |
199 | | |
200 | 56.9k | if (std::ranges::any_of(args, [&block](const auto& elem) { |
201 | 56.9k | return block.get_by_position(elem).column->only_null(); |
202 | 56.9k | })) { |
203 | 4.17k | block.get_by_position(result).column = |
204 | 4.17k | block.get_by_position(result).type->create_column_const(input_rows_count, Field()); |
205 | 4.17k | *executed = true; |
206 | 4.17k | return Status::OK(); |
207 | 4.17k | } |
208 | | |
209 | 18.6k | if (have_null_column(block, args)) { |
210 | 9.24k | bool need_to_default = need_replace_null_data_to_default(); |
211 | | // extract nested column from nulls |
212 | 9.24k | ColumnNumbers new_args; |
213 | 9.24k | Block new_block; |
214 | | |
215 | 33.7k | for (int i = 0; i < args.size(); ++i) { |
216 | 24.4k | uint32_t arg = args[i]; |
217 | 24.4k | new_args.push_back(i); |
218 | 24.4k | new_block.simple_insert(block.get_by_position(arg).unnest_nullable(need_to_default)); |
219 | 24.4k | } |
220 | 9.24k | new_block.simple_insert(block.get_by_position(result)); |
221 | 9.24k | int new_result = new_block.columns() - 1; |
222 | | |
223 | 9.24k | RETURN_IF_ERROR(default_execute(context, new_block, new_args, new_result, block.rows())); |
224 | | // After run with nested, wrap them in null. Before this, block.get_by_position(result).type |
225 | | // is not compatible with get_by_position(result).column |
226 | | |
227 | 9.23k | block.get_by_position(result).column = wrap_in_nullable( |
228 | 9.23k | new_block.get_by_position(new_result).column, block, args, input_rows_count); |
229 | | |
230 | 9.23k | *executed = true; |
231 | 9.23k | return Status::OK(); |
232 | 9.24k | } |
233 | 9.43k | return Status::OK(); |
234 | 18.6k | } |
235 | | |
236 | | Status PreparedFunctionImpl::default_execute(FunctionContext* context, Block& block, |
237 | | const ColumnNumbers& args, uint32_t result, |
238 | 107k | size_t input_rows_count) const { |
239 | 107k | bool executed = false; |
240 | | |
241 | 107k | RETURN_IF_ERROR(default_implementation_for_constant_arguments(context, block, args, result, |
242 | 107k | input_rows_count, &executed)); |
243 | 107k | if (executed) { |
244 | 2.88k | return Status::OK(); |
245 | 2.88k | } |
246 | | |
247 | 104k | return _execute_skipped_constant_deal(context, block, args, result, input_rows_count); |
248 | 107k | } |
249 | | |
250 | | Status PreparedFunctionImpl::execute(FunctionContext* context, Block& block, |
251 | | const ColumnNumbers& args, uint32_t result, |
252 | 98.5k | size_t input_rows_count) const { |
253 | 98.5k | return default_execute(context, block, args, result, input_rows_count); |
254 | 98.5k | } |
255 | | |
256 | 14.6k | void FunctionBuilderImpl::check_number_of_arguments(size_t number_of_arguments) const { |
257 | 14.6k | if (is_variadic()) { |
258 | 4.08k | return; |
259 | 4.08k | } |
260 | | |
261 | 10.5k | size_t expected_number_of_arguments = get_number_of_arguments(); |
262 | | |
263 | 10.5k | DCHECK_EQ(number_of_arguments, expected_number_of_arguments) << fmt::format( |
264 | 0 | "Number of arguments for function {} doesn't match: passed {} , should be {}", |
265 | 0 | get_name(), number_of_arguments, expected_number_of_arguments); |
266 | 10.5k | if (number_of_arguments != expected_number_of_arguments) { |
267 | 0 | throw Exception( |
268 | 0 | ErrorCode::INVALID_ARGUMENT, |
269 | 0 | "Number of arguments for function {} doesn't match: passed {} , should be {}", |
270 | 0 | get_name(), number_of_arguments, expected_number_of_arguments); |
271 | 0 | } |
272 | 10.5k | } |
273 | | |
274 | 14.6k | DataTypePtr FunctionBuilderImpl::get_return_type(const ColumnsWithTypeAndName& arguments) const { |
275 | 14.6k | check_number_of_arguments(arguments.size()); |
276 | | |
277 | 14.6k | if (!arguments.empty() && use_default_implementation_for_nulls()) { |
278 | 13.6k | if (have_null_column(arguments)) { |
279 | 13.4k | ColumnNumbers numbers(arguments.size()); |
280 | 13.4k | std::iota(numbers.begin(), numbers.end(), 0); |
281 | 13.4k | auto [nested_block, _] = |
282 | 13.4k | create_block_with_nested_columns(Block(arguments), numbers, false); |
283 | 13.4k | auto return_type = get_return_type_impl( |
284 | 13.4k | ColumnsWithTypeAndName(nested_block.begin(), nested_block.end())); |
285 | 13.4k | if (!return_type) { |
286 | 0 | return nullptr; |
287 | 0 | } |
288 | 13.4k | return make_nullable(return_type); |
289 | 13.4k | } |
290 | 13.6k | } |
291 | | |
292 | 1.21k | return get_return_type_impl(arguments); |
293 | 14.6k | } |
294 | | |
295 | | bool FunctionBuilderImpl::is_date_or_datetime_or_decimal( |
296 | 57 | const DataTypePtr& return_type, const DataTypePtr& func_return_type) const { |
297 | 57 | return (is_date_or_datetime(return_type->get_primitive_type()) && |
298 | 57 | is_date_or_datetime(func_return_type->get_primitive_type())) || |
299 | 57 | (is_date_v2_or_datetime_v2(return_type->get_primitive_type()) && |
300 | 57 | is_date_v2_or_datetime_v2(func_return_type->get_primitive_type())) || |
301 | | // For some date functions such as str_to_date(string, string), return_type will |
302 | | // be datetimev2 if users enable datev2 but get_return_type(arguments) will still |
303 | | // return datetime. We need keep backward compatibility here. |
304 | 57 | (is_date_v2_or_datetime_v2(return_type->get_primitive_type()) && |
305 | 26 | is_date_or_datetime(func_return_type->get_primitive_type())) || |
306 | 57 | (is_date_or_datetime(return_type->get_primitive_type()) && |
307 | 26 | is_date_v2_or_datetime_v2(func_return_type->get_primitive_type())) || |
308 | 57 | (is_decimal(return_type->get_primitive_type()) && |
309 | 26 | is_decimal(func_return_type->get_primitive_type())) || |
310 | 57 | (is_time_type(return_type->get_primitive_type()) && |
311 | 24 | is_time_type(func_return_type->get_primitive_type())); |
312 | 57 | } |
313 | | |
314 | 20 | bool contains_date_or_datetime_or_decimal(const DataTypePtr& type) { |
315 | 20 | auto type_ptr = type->is_nullable() ? ((DataTypeNullable*)type.get())->get_nested_type() : type; |
316 | | |
317 | 20 | switch (type_ptr->get_primitive_type()) { |
318 | 0 | case TYPE_ARRAY: { |
319 | 0 | const auto* array_type = assert_cast<const DataTypeArray*>(type_ptr.get()); |
320 | 0 | return contains_date_or_datetime_or_decimal(array_type->get_nested_type()); |
321 | 0 | } |
322 | 0 | case TYPE_MAP: { |
323 | 0 | const auto* map_type = assert_cast<const DataTypeMap*>(type_ptr.get()); |
324 | 0 | return contains_date_or_datetime_or_decimal(map_type->get_key_type()) || |
325 | 0 | contains_date_or_datetime_or_decimal(map_type->get_value_type()); |
326 | 0 | } |
327 | 0 | case TYPE_STRUCT: { |
328 | 0 | const auto* struct_type = assert_cast<const DataTypeStruct*>(type_ptr.get()); |
329 | 0 | const auto& elements = struct_type->get_elements(); |
330 | 0 | return std::ranges::any_of(elements, [](const DataTypePtr& element) { |
331 | 0 | return contains_date_or_datetime_or_decimal(element); |
332 | 0 | }); |
333 | 0 | } |
334 | 20 | default: |
335 | | // For scalar types, check if it's date/datetime/decimal |
336 | 20 | return is_date_or_datetime(type_ptr->get_primitive_type()) || |
337 | 20 | is_date_v2_or_datetime_v2(type_ptr->get_primitive_type()) || |
338 | 20 | is_decimal(type_ptr->get_primitive_type()) || |
339 | 20 | is_time_type(type_ptr->get_primitive_type()); |
340 | 20 | } |
341 | 20 | } |
342 | | |
343 | | // make sure array/map/struct and nested array/map/struct can be check |
344 | | bool FunctionBuilderImpl::is_nested_type_date_or_datetime_or_decimal( |
345 | 22 | const DataTypePtr& return_type, const DataTypePtr& func_return_type) const { |
346 | 22 | auto return_type_ptr = return_type->is_nullable() |
347 | 22 | ? ((DataTypeNullable*)return_type.get())->get_nested_type() |
348 | 22 | : return_type; |
349 | 22 | auto func_return_type_ptr = |
350 | 22 | func_return_type->is_nullable() |
351 | 22 | ? ((DataTypeNullable*)func_return_type.get())->get_nested_type() |
352 | 22 | : func_return_type; |
353 | | // make sure that map/struct/array also need to check |
354 | 22 | if (return_type_ptr->get_primitive_type() != func_return_type_ptr->get_primitive_type()) { |
355 | 2 | return false; |
356 | 2 | } |
357 | | |
358 | | // Check if this type contains date/datetime/decimal types |
359 | 20 | if (!contains_date_or_datetime_or_decimal(return_type_ptr)) { |
360 | | // If no date/datetime/decimal types, just pass through |
361 | 20 | return true; |
362 | 20 | } |
363 | | |
364 | | // If contains date/datetime/decimal types, recursively check each element |
365 | 0 | switch (return_type_ptr->get_primitive_type()) { |
366 | 0 | case TYPE_ARRAY: { |
367 | 0 | auto nested_return_type = remove_nullable( |
368 | 0 | (assert_cast<const DataTypeArray*>(return_type_ptr.get()))->get_nested_type()); |
369 | 0 | auto nested_func_type = remove_nullable( |
370 | 0 | (assert_cast<const DataTypeArray*>(func_return_type_ptr.get()))->get_nested_type()); |
371 | 0 | return is_nested_type_date_or_datetime_or_decimal(nested_return_type, nested_func_type); |
372 | 0 | } |
373 | 0 | case TYPE_MAP: { |
374 | 0 | const auto* return_map = assert_cast<const DataTypeMap*>(return_type_ptr.get()); |
375 | 0 | const auto* func_map = assert_cast<const DataTypeMap*>(func_return_type_ptr.get()); |
376 | |
|
377 | 0 | auto key_return = remove_nullable(return_map->get_key_type()); |
378 | 0 | auto key_func = remove_nullable(func_map->get_key_type()); |
379 | 0 | auto value_return = remove_nullable(return_map->get_value_type()); |
380 | 0 | auto value_func = remove_nullable(func_map->get_value_type()); |
381 | |
|
382 | 0 | return is_nested_type_date_or_datetime_or_decimal(key_return, key_func) && |
383 | 0 | is_nested_type_date_or_datetime_or_decimal(value_return, value_func); |
384 | 0 | } |
385 | 0 | case TYPE_STRUCT: { |
386 | 0 | const auto* return_struct = assert_cast<const DataTypeStruct*>(return_type_ptr.get()); |
387 | 0 | const auto* func_struct = assert_cast<const DataTypeStruct*>(func_return_type_ptr.get()); |
388 | |
|
389 | 0 | auto return_elements = return_struct->get_elements(); |
390 | 0 | auto func_elements = func_struct->get_elements(); |
391 | |
|
392 | 0 | if (return_elements.size() != func_elements.size()) { |
393 | 0 | return false; |
394 | 0 | } |
395 | | |
396 | 0 | for (size_t i = 0; i < return_elements.size(); i++) { |
397 | 0 | auto elem_return = remove_nullable(return_elements[i]); |
398 | 0 | auto elem_func = remove_nullable(func_elements[i]); |
399 | |
|
400 | 0 | if (!is_nested_type_date_or_datetime_or_decimal(elem_return, elem_func)) { |
401 | 0 | return false; |
402 | 0 | } |
403 | 0 | } |
404 | 0 | return true; |
405 | 0 | } |
406 | 0 | default: |
407 | 0 | return is_date_or_datetime_or_decimal(return_type_ptr, func_return_type_ptr); |
408 | 0 | } |
409 | 0 | } |
410 | | |
411 | | ZoneMapFilterResult IFunctionBase::evaluate_zonemap_filter( |
412 | 0 | const ZoneMapEvalContext& ctx, const VExprSPtrs& function_arguments) const { |
413 | 0 | return unsupported_zonemap_filter(ctx); |
414 | 0 | } |
415 | | |
416 | | ZoneMapFilterResult IFunctionBase::evaluate_dictionary_filter( |
417 | 0 | const DictionaryEvalContext& ctx, const VExprSPtrs& function_arguments) const { |
418 | 0 | return ZoneMapFilterResult::kUnsupported; |
419 | 0 | } |
420 | | |
421 | | ZoneMapFilterResult IFunctionBase::evaluate_bloom_filter( |
422 | 0 | const BloomFilterEvalContext& ctx, const VExprSPtrs& function_arguments) const { |
423 | 0 | return ZoneMapFilterResult::kUnsupported; |
424 | 0 | } |
425 | | |
426 | | } // namespace doris |