Coverage Report

Created: 2026-07-30 12:27

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exprs/vexpr.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "exprs/vexpr.h"
19
20
#include <fmt/format.h>
21
#include <gen_cpp/Exprs_types.h>
22
#include <gen_cpp/FrontendService_types.h>
23
#include <thrift/protocol/TDebugProtocol.h>
24
25
#include <algorithm>
26
#include <boost/algorithm/string/split.hpp>
27
#include <boost/iterator/iterator_facade.hpp>
28
#include <cstdint>
29
#include <memory>
30
#include <optional>
31
#include <stack>
32
#include <string_view>
33
#include <unordered_set>
34
#include <utility>
35
36
#include "common/config.h"
37
#include "common/exception.h"
38
#include "common/status.h"
39
#include "core/column/column_nothing.h"
40
#include "core/column/column_vector.h"
41
#include "core/data_type/data_type_array.h"
42
#include "core/data_type/data_type_decimal.h"
43
#include "core/data_type/data_type_factory.hpp"
44
#include "core/data_type/data_type_nullable.h"
45
#include "core/data_type/data_type_number.h"
46
#include "core/data_type/define_primitive_type.h"
47
#include "core/field.h"
48
#include "core/string_ref.h"
49
#include "core/value/timestamptz_value.h"
50
#include "exec/common/util.hpp"
51
#include "exec/pipeline/pipeline_task.h"
52
#include "exprs/short_circuit_evaluation_expr.h"
53
#include "exprs/varray_literal.h"
54
#include "exprs/vcase_expr.h"
55
#include "exprs/vcast_expr.h"
56
#include "exprs/vcolumn_ref.h"
57
#include "exprs/vcompound_pred.h"
58
#include "exprs/vcondition_expr.h"
59
#include "exprs/vectorized_fn_call.h"
60
#include "exprs/vexpr_context.h"
61
#include "exprs/vexpr_fwd.h"
62
#include "exprs/vin_predicate.h"
63
#include "exprs/vinfo_func.h"
64
#include "exprs/virtual_slot_ref.h"
65
#include "exprs/vlambda_function_call_expr.h"
66
#include "exprs/vlambda_function_expr.h"
67
#include "exprs/vliteral.h"
68
#include "exprs/vmap_literal.h"
69
#include "exprs/vmatch_predicate.h"
70
#include "exprs/vsearch.h"
71
#include "exprs/vslot_ref.h"
72
#include "exprs/vstruct_literal.h"
73
#include "storage/index/ann/ann_search_params.h"
74
#include "storage/index/ann/ann_topn_runtime.h"
75
#include "storage/index/inverted/inverted_index_parser.h"
76
#include "storage/index/zone_map/zonemap_eval_context.h"
77
#include "storage/segment/column_reader.h"
78
79
namespace doris {
80
81
class RowDescriptor;
82
class RuntimeState;
83
84
0
ZoneMapFilterResult VExpr::evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const {
85
0
    return unsupported_zonemap_filter(ctx);
86
0
}
87
88
0
ZoneMapFilterResult VExpr::evaluate_dictionary_filter(const DictionaryEvalContext& ctx) const {
89
0
    return ZoneMapFilterResult::kUnsupported;
90
0
}
91
92
0
ZoneMapFilterResult VExpr::evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const {
93
0
    return ZoneMapFilterResult::kUnsupported;
94
0
}
95
96
// NOLINTBEGIN(readability-function-cognitive-complexity)
97
// NOLINTBEGIN(readability-function-size)
98
TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, int precision,
99
3.26M
                                 int scale) {
100
3.26M
    TExprNode node;
101
102
3.26M
    switch (type) {
103
161
    case TYPE_BOOLEAN: {
104
161
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_BOOLEAN>(data, &node));
105
161
        break;
106
161
    }
107
715
    case TYPE_TINYINT: {
108
715
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_TINYINT>(data, &node));
109
715
        break;
110
715
    }
111
1.07k
    case TYPE_SMALLINT: {
112
1.07k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_SMALLINT>(data, &node));
113
1.07k
        break;
114
1.07k
    }
115
3.25M
    case TYPE_INT: {
116
3.25M
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_INT>(data, &node));
117
3.25M
        break;
118
3.25M
    }
119
3.25M
    case TYPE_BIGINT: {
120
8.15k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_BIGINT>(data, &node));
121
8.15k
        break;
122
8.15k
    }
123
8.15k
    case TYPE_LARGEINT: {
124
122
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_LARGEINT>(data, &node));
125
122
        break;
126
122
    }
127
206
    case TYPE_FLOAT: {
128
206
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_FLOAT>(data, &node));
129
206
        break;
130
206
    }
131
206
    case TYPE_DOUBLE: {
132
53
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DOUBLE>(data, &node));
133
53
        break;
134
53
    }
135
709
    case TYPE_DATEV2: {
136
709
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATEV2>(data, &node));
137
709
        break;
138
709
    }
139
947
    case TYPE_DATETIMEV2: {
140
947
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATETIMEV2>(data, &node, precision, scale));
141
947
        break;
142
947
    }
143
947
    case TYPE_DATE: {
144
32
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATE>(data, &node));
145
32
        break;
146
32
    }
147
54
    case TYPE_DATETIME: {
148
54
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATETIME>(data, &node));
149
54
        break;
150
54
    }
151
54
    case TYPE_DECIMALV2: {
152
6
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DECIMALV2>(data, &node, precision, scale));
153
6
        break;
154
6
    }
155
65
    case TYPE_DECIMAL32: {
156
65
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DECIMAL32>(data, &node, precision, scale));
157
65
        break;
158
65
    }
159
780
    case TYPE_DECIMAL64: {
160
780
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DECIMAL64>(data, &node, precision, scale));
161
780
        break;
162
780
    }
163
780
    case TYPE_DECIMAL128I: {
164
99
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DECIMAL128I>(data, &node, precision, scale));
165
99
        break;
166
99
    }
167
99
    case TYPE_DECIMAL256: {
168
46
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DECIMAL256>(data, &node, precision, scale));
169
46
        break;
170
46
    }
171
46
    case TYPE_CHAR: {
172
16
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_CHAR>(data, &node));
173
16
        break;
174
16
    }
175
552
    case TYPE_VARCHAR: {
176
552
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_VARCHAR>(data, &node));
177
552
        break;
178
552
    }
179
1.25k
    case TYPE_STRING: {
180
1.25k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_STRING>(data, &node));
181
1.25k
        break;
182
1.25k
    }
183
1.25k
    case TYPE_VARBINARY: {
184
0
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_VARBINARY>(data, &node));
185
0
        break;
186
0
    }
187
30
    case TYPE_IPV4: {
188
30
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV4>(data, &node));
189
30
        break;
190
30
    }
191
30
    case TYPE_IPV6: {
192
18
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV6>(data, &node));
193
18
        break;
194
18
    }
195
18
    case TYPE_TIMEV2: {
196
0
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_TIMEV2>(data, &node, precision, scale));
197
0
        break;
198
0
    }
199
84
    case TYPE_TIMESTAMPTZ: {
200
84
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_TIMESTAMPTZ>(data, &node, precision, scale));
201
84
        break;
202
84
    }
203
84
    default:
204
0
        throw Exception(ErrorCode::INTERNAL_ERROR, "runtime filter meet invalid type {}",
205
0
                        int(type));
206
3.26M
    }
207
3.25M
    return node;
208
3.26M
}
209
210
TExprNode create_texpr_node_from(const Field& field, const PrimitiveType& type, int precision,
211
5.80k
                                 int scale) {
212
5.80k
    TExprNode node;
213
5.80k
    switch (type) {
214
1
    case TYPE_BOOLEAN: {
215
1
        const auto& storage = static_cast<bool>(field.get<TYPE_BOOLEAN>());
216
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_BOOLEAN>(&storage, &node));
217
1
        break;
218
1
    }
219
1
    case TYPE_TINYINT: {
220
0
        const auto& storage = static_cast<int8_t>(field.get<TYPE_TINYINT>());
221
0
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_TINYINT>(&storage, &node));
222
0
        break;
223
0
    }
224
1
    case TYPE_SMALLINT: {
225
1
        const auto& storage = static_cast<int16_t>(field.get<TYPE_SMALLINT>());
226
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_SMALLINT>(&storage, &node));
227
1
        break;
228
1
    }
229
1.08k
    case TYPE_INT: {
230
1.08k
        const auto& storage = static_cast<int32_t>(field.get<TYPE_INT>());
231
1.08k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_INT>(&storage, &node));
232
1.08k
        break;
233
1.08k
    }
234
1.08k
    case TYPE_BIGINT: {
235
45
        const auto& storage = static_cast<int64_t>(field.get<TYPE_BIGINT>());
236
45
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_BIGINT>(&storage, &node));
237
45
        break;
238
45
    }
239
45
    case TYPE_LARGEINT: {
240
1
        const auto& storage = static_cast<int128_t>(field.get<TYPE_LARGEINT>());
241
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_LARGEINT>(&storage, &node));
242
1
        break;
243
1
    }
244
1
    case TYPE_FLOAT: {
245
1
        const auto& storage = static_cast<float>(field.get<TYPE_FLOAT>());
246
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_FLOAT>(&storage, &node));
247
1
        break;
248
1
    }
249
1
    case TYPE_DOUBLE: {
250
1
        const auto& storage = static_cast<double>(field.get<TYPE_DOUBLE>());
251
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DOUBLE>(&storage, &node));
252
1
        break;
253
1
    }
254
1
    case TYPE_DATEV2: {
255
1
        const auto& storage = field.get<TYPE_DATEV2>();
256
257
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATEV2>(&storage, &node));
258
1
        break;
259
1
    }
260
2
    case TYPE_DATETIMEV2: {
261
2
        const auto& storage = field.get<TYPE_DATETIMEV2>();
262
2
        THROW_IF_ERROR(
263
2
                create_texpr_literal_node<TYPE_DATETIMEV2>(&storage, &node, precision, scale));
264
2
        break;
265
2
    }
266
2
    case TYPE_TIMESTAMPTZ: {
267
1
        const auto& storage = field.get<TYPE_TIMESTAMPTZ>();
268
269
1
        THROW_IF_ERROR(
270
1
                create_texpr_literal_node<TYPE_TIMESTAMPTZ>(&storage, &node, precision, scale));
271
1
        break;
272
1
    }
273
1
    case TYPE_DATE: {
274
1
        const auto& storage = field.get<TYPE_DATE>();
275
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATE>(&storage, &node));
276
1
        break;
277
1
    }
278
1
    case TYPE_DATETIME: {
279
1
        const auto& storage = field.get<TYPE_DATETIME>();
280
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATETIME>(&storage, &node));
281
1
        break;
282
1
    }
283
1
    case TYPE_DECIMALV2: {
284
1
        const auto& storage = field.get<TYPE_DECIMALV2>();
285
286
1
        THROW_IF_ERROR(
287
1
                create_texpr_literal_node<TYPE_DECIMALV2>(&storage, &node, precision, scale));
288
1
        break;
289
1
    }
290
2
    case TYPE_DECIMAL32: {
291
2
        const auto& storage = field.get<TYPE_DECIMAL32>();
292
2
        THROW_IF_ERROR(
293
2
                create_texpr_literal_node<TYPE_DECIMAL32>(&storage, &node, precision, scale));
294
2
        break;
295
2
    }
296
2
    case TYPE_DECIMAL64: {
297
2
        const auto& storage = field.get<TYPE_DECIMAL64>();
298
2
        THROW_IF_ERROR(
299
2
                create_texpr_literal_node<TYPE_DECIMAL64>(&storage, &node, precision, scale));
300
2
        break;
301
2
    }
302
2
    case TYPE_DECIMAL128I: {
303
2
        const auto& storage = field.get<TYPE_DECIMAL128I>();
304
2
        THROW_IF_ERROR(
305
2
                create_texpr_literal_node<TYPE_DECIMAL128I>(&storage, &node, precision, scale));
306
2
        break;
307
2
    }
308
2
    case TYPE_DECIMAL256: {
309
2
        const auto& storage = field.get<TYPE_DECIMAL256>();
310
2
        THROW_IF_ERROR(
311
2
                create_texpr_literal_node<TYPE_DECIMAL256>(&storage, &node, precision, scale));
312
2
        break;
313
2
    }
314
285
    case TYPE_CHAR: {
315
285
        const auto& storage = field.get<TYPE_CHAR>();
316
285
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_CHAR>(&storage, &node));
317
285
        break;
318
285
    }
319
1.35k
    case TYPE_VARCHAR: {
320
1.35k
        const auto& storage = field.get<TYPE_VARCHAR>();
321
1.35k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_VARCHAR>(&storage, &node));
322
1.35k
        break;
323
1.35k
    }
324
3.01k
    case TYPE_STRING: {
325
3.01k
        const auto& storage = field.get<TYPE_STRING>();
326
3.01k
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_STRING>(&storage, &node));
327
3.01k
        break;
328
3.01k
    }
329
3.01k
    case TYPE_IPV4: {
330
0
        const auto& storage = field.get<TYPE_IPV4>();
331
0
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV4>(&storage, &node));
332
0
        break;
333
0
    }
334
0
    case TYPE_IPV6: {
335
0
        const auto& storage = field.get<TYPE_IPV6>();
336
0
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV6>(&storage, &node));
337
0
        break;
338
0
    }
339
1
    case TYPE_TIMEV2: {
340
1
        const auto& storage = field.get<TYPE_TIMEV2>();
341
1
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_TIMEV2>(&storage, &node));
342
1
        break;
343
1
    }
344
2
    case TYPE_VARBINARY: {
345
2
        const auto& svf = field.get<TYPE_VARBINARY>();
346
2
        THROW_IF_ERROR(create_texpr_literal_node<TYPE_VARBINARY>(&svf, &node));
347
2
        break;
348
2
    }
349
2
    default:
350
0
        throw Exception(ErrorCode::INTERNAL_ERROR, "runtime filter meet invalid type {}",
351
0
                        int(type));
352
5.80k
    }
353
5.80k
    return node;
354
5.80k
}
355
356
// NOLINTEND(readability-function-size)
357
// NOLINTEND(readability-function-cognitive-complexity)
358
} // namespace doris
359
360
namespace doris {
361
362
namespace {
363
364
// Collect all slot ids referenced anywhere in `expr` (recursing children).
365
57
void collect_referenced_slot_ids(const VExpr& expr, std::unordered_set<int>& slot_ids) {
366
57
    if (expr.node_type() == TExprNodeType::SLOT_REF) {
367
42
        slot_ids.insert(static_cast<const VSlotRef&>(expr).slot_id());
368
42
        return;
369
42
    }
370
16
    for (const auto& child : expr.children()) {
371
16
        collect_referenced_slot_ids(*child, slot_ids);
372
16
    }
373
15
}
374
375
// Whether the value side of a predicate derives from exactly the single slot `slot_id`
376
// (at least one reference to it, and no reference to any other slot).
377
41
bool derives_from_single_slot(const VExpr& value_side, int slot_id) {
378
41
    std::unordered_set<int> slot_ids;
379
41
    collect_referenced_slot_ids(value_side, slot_ids);
380
41
    return slot_ids.size() == 1 && slot_ids.contains(slot_id);
381
41
}
382
383
// Known-unsafe builtin function names that must never be dict-filtered regardless of the
384
// planner verdict: either non-deterministic (evaluated once per distinct value and cached,
385
// so a per-row-varying result would be wrong) or NULL-breaking (can turn a NULL input into a
386
// non-NULL output, so NULL rows -- which have no dictionary entry -- would be wrongly kept or
387
// dropped). This is a defense-in-depth veto: the planner is authoritative and knows UDFs, but
388
// a planner bug or FE/BE version skew must not silently produce wrong results for these.
389
6
bool is_dict_unsafe_function_name(const std::string& fn_name) {
390
6
    static const std::unordered_set<std::string> kDictUnsafe = {
391
            // NULL-breaking: map a NULL argument to a non-NULL result
392
6
            "coalesce", "ifnull", "nvl", "nvl2", "if", "nullif", "concat_ws", "ifnotnull",
393
            // non-deterministic
394
6
            "random", "rand", "uuid", "uuid_numeric", "now", "current_timestamp", "curdate",
395
6
            "current_date", "curtime", "current_time", "unix_timestamp", "utc_timestamp"};
396
6
    return kDictUnsafe.contains(fn_name);
397
6
}
398
399
// Whether `expr` references any known-unsafe function anywhere in its tree.
400
44
bool references_dict_unsafe_function(const VExpr& expr) {
401
44
    if ((expr.node_type() == TExprNodeType::FUNCTION_CALL ||
402
44
         expr.node_type() == TExprNodeType::COMPUTE_FUNCTION_CALL) &&
403
44
        is_dict_unsafe_function_name(expr.fn().name.function_name)) {
404
3
        return true;
405
3
    }
406
41
    const auto& children = expr.children();
407
41
    return std::any_of(children.begin(), children.end(), [](const VExprSPtr& child) {
408
6
        return references_dict_unsafe_function(*child);
409
6
    });
410
44
}
411
412
// Whether `value_side` is a bare column ref to `slot_id` (the pre-expression form of
413
// dict filtering: `col = 'x'` / `col IN (...)`). This is the only form BE dict-filters
414
// without a planner verdict, so an older FE that never stamped can_dict_filter keeps the
415
// pre-existing bare-column optimization but never gets the new expression path.
416
12
bool is_bare_slot_ref(const VExpr& value_side, int slot_id) {
417
12
    return value_side.is_slot_ref() &&
418
12
           static_cast<const VSlotRef&>(value_side).slot_id() == slot_id;
419
12
}
420
421
} // namespace
422
423
58
bool VExpr::can_push_down_to_dict_filter(const VExprSPtr& root, int slot_id, bool allow_expr) {
424
58
    if (root == nullptr) {
425
1
        return false;
426
1
    }
427
    // Only equality (BINARY_PRED with opcode EQ) and IN are rewritten into a dict-code
428
    // predicate downstream. Reject other binary opcodes (<, >, !=) explicitly so the
429
    // structural shape here matches what _rewrite_dict_conjuncts actually produces.
430
57
    if (root->node_type() == TExprNodeType::BINARY_PRED) {
431
55
        if (root->op() != TExprOpcode::EQ) {
432
2
            return false;
433
2
        }
434
55
    } else if (root->node_type() != TExprNodeType::IN_PRED) {
435
1
        return false;
436
1
    }
437
54
    const auto& children = root->children();
438
54
    if (children.empty()) {
439
0
        return false;
440
0
    }
441
    // The predicate's first child is the value side; the rest are literals.
442
108
    for (size_t i = 1; i < children.size(); ++i) {
443
54
        if (!children[i]->is_constant()) {
444
0
            return false;
445
0
        }
446
54
    }
447
    // When expression dict-filtering is disabled (session variable off), or the planner
448
    // did not stamp a verdict (older FE), only the original form -- a bare column ref
449
    // compared to constants -- is allowed. The expression path is planner-authoritative:
450
    // determinism, NULL propagation, and UDF semantics all live in the FE, so BE has no
451
    // safe way to decide on its own for a complex value side.
452
54
    std::optional<bool> planner_verdict = root->can_dict_filter_from_planner();
453
54
    if (!allow_expr || !planner_verdict.has_value()) {
454
12
        return is_bare_slot_ref(*children[0], slot_id);
455
12
    }
456
    // Planner stamped a verdict: trust it, but still confirm here that the value side
457
    // derives from exactly this slot (dictionaries are per column) and does not reference
458
    // a known-unsafe builtin (defense-in-depth against a planner bug or FE/BE version
459
    // skew).
460
42
    return *planner_verdict && derives_from_single_slot(*children[0], slot_id) &&
461
42
           !references_dict_unsafe_function(*children[0]);
462
54
}
463
464
VExpr::VExpr(const TExprNode& node)
465
13.3M
        : _node_type(node.node_type),
466
13.3M
          _opcode(node.__isset.opcode ? node.opcode : TExprOpcode::INVALID_OPCODE) {
467
13.3M
    if (node.__isset.fn) {
468
918k
        _fn = node.fn;
469
918k
    }
470
471
13.3M
    if (node.__isset.can_dict_filter) {
472
468k
        _can_dict_filter_from_planner = node.can_dict_filter;
473
468k
    }
474
475
13.3M
    bool is_nullable = true;
476
13.3M
    if (node.__isset.is_nullable) {
477
10.0M
        is_nullable = node.is_nullable;
478
10.0M
    }
479
    // If we define null literal ,should make nullable data type to get correct field instead of undefined ptr
480
13.3M
    if (node.node_type == TExprNodeType::NULL_LITERAL) {
481
426k
        CHECK(is_nullable);
482
426k
    }
483
13.3M
    _data_type = get_data_type_with_default_argument(
484
13.3M
            DataTypeFactory::instance().create_data_type(node.type, is_nullable));
485
13.3M
}
486
487
345k
VExpr::VExpr(const VExpr& vexpr) = default;
488
489
VExpr::VExpr(DataTypePtr type, bool is_slotref)
490
1.58M
        : _opcode(TExprOpcode::INVALID_OPCODE),
491
1.58M
          _data_type(get_data_type_with_default_argument(type)) {
492
1.58M
    if (is_slotref) {
493
1.19M
        _node_type = TExprNodeType::SLOT_REF;
494
1.19M
    }
495
1.58M
}
496
497
121k
TExprNode VExpr::clone_texpr_node() const {
498
121k
    TExprNode node;
499
121k
    node.__set_node_type(_node_type);
500
121k
    node.__set_opcode(_opcode);
501
121k
    node.__set_type(create_type_desc(remove_nullable(_data_type)->get_primitive_type(),
502
121k
                                     static_cast<int>(_data_type->get_precision()),
503
121k
                                     static_cast<int>(_data_type->get_scale())));
504
121k
    node.__set_is_nullable(_data_type->is_nullable());
505
121k
    node.__set_num_children(get_num_children());
506
121k
    node.__set_fn(_fn);
507
121k
    return node;
508
121k
}
509
510
0
Status VExpr::clone_node(VExprSPtr* cloned_expr) const {
511
0
    DORIS_CHECK(cloned_expr != nullptr);
512
0
    return Status::NotSupported("Cannot clone expression {} for file-local rewrite", expr_name());
513
0
}
514
515
Status VExpr::deep_clone(VExprSPtr* cloned_expr,
516
1.17M
                         const VExprCloneNodeOverride& clone_node_override) const {
517
1.17M
    DORIS_CHECK(cloned_expr != nullptr);
518
519
1.17M
    VExprSPtr cloned;
520
1.17M
    if (clone_node_override) {
521
1.07M
        RETURN_IF_ERROR(clone_node_override(*this, &cloned));
522
1.07M
    }
523
1.17M
    if (cloned == nullptr) {
524
1.16M
        RETURN_IF_ERROR(clone_node(&cloned));
525
1.16M
    }
526
1.17M
    DORIS_CHECK(cloned != nullptr);
527
528
1.17M
    VExprSPtrs cloned_children;
529
1.17M
    cloned_children.reserve(_children.size());
530
1.17M
    for (const auto& child : _children) {
531
776k
        DORIS_CHECK(child != nullptr);
532
776k
        VExprSPtr cloned_child;
533
776k
        RETURN_IF_ERROR(child->deep_clone(&cloned_child, clone_node_override));
534
776k
        cloned_children.push_back(std::move(cloned_child));
535
776k
    }
536
1.17M
    cloned->set_children(std::move(cloned_children));
537
1.17M
    cloned->reset_prepare_state();
538
1.17M
    *cloned_expr = std::move(cloned);
539
1.17M
    return Status::OK();
540
1.17M
}
541
542
9.64M
Status VExpr::prepare(RuntimeState* state, const RowDescriptor& row_desc, VExprContext* context) {
543
9.64M
    ++context->_depth_num;
544
9.64M
    if (context->_depth_num > config::max_depth_of_expr_tree) {
545
0
        return Status::Error<ErrorCode::EXCEEDED_LIMIT>(
546
0
                "The depth of the expression tree is too big, make it less than {}",
547
0
                config::max_depth_of_expr_tree);
548
0
    }
549
550
9.64M
    for (auto& i : _children) {
551
2.12M
        RETURN_IF_ERROR(i->prepare(state, row_desc, context));
552
2.12M
    }
553
9.64M
    --context->_depth_num;
554
9.64M
#ifndef BE_TEST
555
9.64M
    _enable_inverted_index_query = state->query_options().enable_inverted_index_query;
556
9.64M
#endif
557
9.64M
    return Status::OK();
558
9.64M
}
559
560
Status VExpr::open(RuntimeState* state, VExprContext* context,
561
32.9M
                   FunctionContext::FunctionStateScope scope) {
562
32.9M
    for (auto& i : _children) {
563
505k
        RETURN_IF_ERROR(i->open(state, context, scope));
564
505k
    }
565
32.9M
    if (scope == FunctionContext::FRAGMENT_LOCAL) {
566
9.18M
        RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr));
567
9.18M
    }
568
32.9M
    return Status::OK();
569
32.9M
}
570
571
2.14M
void VExpr::reset_prepare_state() {
572
2.14M
    _prepared = false;
573
2.14M
    _prepare_finished = false;
574
2.14M
    _open_finished = false;
575
2.14M
    for (auto& child : _children) {
576
968k
        child->reset_prepare_state();
577
968k
    }
578
2.14M
}
579
580
35.9M
void VExpr::close(VExprContext* context, FunctionContext::FunctionStateScope scope) {
581
35.9M
    for (auto& i : _children) {
582
5.28M
        i->close(context, scope);
583
5.28M
    }
584
35.9M
}
585
586
// NOLINTBEGIN(readability-function-size)
587
9.88M
Status VExpr::create_expr(const TExprNode& expr_node, VExprSPtr& expr) {
588
9.88M
    try {
589
9.88M
        switch (expr_node.node_type) {
590
24.2k
        case TExprNodeType::BOOL_LITERAL:
591
869k
        case TExprNodeType::INT_LITERAL:
592
917k
        case TExprNodeType::LARGE_INT_LITERAL:
593
919k
        case TExprNodeType::IPV4_LITERAL:
594
921k
        case TExprNodeType::IPV6_LITERAL:
595
969k
        case TExprNodeType::FLOAT_LITERAL:
596
1.03M
        case TExprNodeType::DECIMAL_LITERAL:
597
1.21M
        case TExprNodeType::DATE_LITERAL:
598
1.21M
        case TExprNodeType::TIMEV2_LITERAL:
599
2.04M
        case TExprNodeType::STRING_LITERAL:
600
2.04M
        case TExprNodeType::JSON_LITERAL:
601
2.04M
        case TExprNodeType::VARBINARY_LITERAL:
602
2.47M
        case TExprNodeType::NULL_LITERAL: {
603
2.47M
            expr = VLiteral::create_shared(expr_node);
604
2.47M
            break;
605
2.04M
        }
606
18.8k
        case TExprNodeType::ARRAY_LITERAL: {
607
18.8k
            expr = VArrayLiteral::create_shared(expr_node);
608
18.8k
            break;
609
2.04M
        }
610
12.0k
        case TExprNodeType::MAP_LITERAL: {
611
12.0k
            expr = VMapLiteral::create_shared(expr_node);
612
12.0k
            break;
613
2.04M
        }
614
1.60k
        case TExprNodeType::STRUCT_LITERAL: {
615
1.60k
            expr = VStructLiteral::create_shared(expr_node);
616
1.60k
            break;
617
2.04M
        }
618
6.44M
        case TExprNodeType::SLOT_REF: {
619
6.44M
            if (expr_node.slot_ref.__isset.is_virtual_slot && expr_node.slot_ref.is_virtual_slot) {
620
260
                expr = VirtualSlotRef::create_shared(expr_node);
621
260
                expr->_node_type = TExprNodeType::VIRTUAL_SLOT_REF;
622
6.44M
            } else {
623
6.44M
                expr = VSlotRef::create_shared(expr_node);
624
6.44M
            }
625
6.44M
            break;
626
2.04M
        }
627
1.34k
        case TExprNodeType::COLUMN_REF: {
628
1.34k
            expr = VColumnRef::create_shared(expr_node);
629
1.34k
            break;
630
2.04M
        }
631
6.51k
        case TExprNodeType::COMPOUND_PRED: {
632
6.51k
            expr = VCompoundPred::create_shared(expr_node);
633
6.51k
            break;
634
2.04M
        }
635
1.02k
        case TExprNodeType::LAMBDA_FUNCTION_EXPR: {
636
1.02k
            expr = VLambdaFunctionExpr::create_shared(expr_node);
637
1.02k
            break;
638
2.04M
        }
639
1.02k
        case TExprNodeType::LAMBDA_FUNCTION_CALL_EXPR: {
640
1.02k
            expr = VLambdaFunctionCallExpr::create_shared(expr_node);
641
1.02k
            break;
642
2.04M
        }
643
89.6k
        case TExprNodeType::ARITHMETIC_EXPR:
644
587k
        case TExprNodeType::BINARY_PRED:
645
587k
        case TExprNodeType::NULL_AWARE_BINARY_PRED:
646
587k
        case TExprNodeType::COMPUTE_FUNCTION_CALL: {
647
587k
            expr = VectorizedFnCall::create_shared(expr_node);
648
587k
            break;
649
587k
        }
650
192k
        case TExprNodeType::FUNCTION_CALL: {
651
192k
            if (expr_node.fn.name.function_name == "if") {
652
5.33k
                if (expr_node.__isset.short_circuit_evaluation &&
653
5.33k
                    expr_node.short_circuit_evaluation) {
654
879
                    expr = ShortCircuitIfExpr::create_shared(expr_node);
655
4.46k
                } else {
656
4.46k
                    expr = VectorizedIfExpr::create_shared(expr_node);
657
4.46k
                }
658
5.33k
                break;
659
187k
            } else if (expr_node.fn.name.function_name == "ifnull" ||
660
187k
                       expr_node.fn.name.function_name == "nvl") {
661
2.67k
                if (expr_node.__isset.short_circuit_evaluation &&
662
2.67k
                    expr_node.short_circuit_evaluation) {
663
263
                    expr = ShortCircuitIfNullExpr::create_shared(expr_node);
664
2.41k
                } else {
665
2.41k
                    expr = VectorizedIfNullExpr::create_shared(expr_node);
666
2.41k
                }
667
2.67k
                break;
668
184k
            } else if (expr_node.fn.name.function_name == "coalesce") {
669
170
                if (expr_node.__isset.short_circuit_evaluation &&
670
170
                    expr_node.short_circuit_evaluation) {
671
109
                    expr = ShortCircuitCoalesceExpr::create_shared(expr_node);
672
109
                } else {
673
61
                    expr = VectorizedCoalesceExpr::create_shared(expr_node);
674
61
                }
675
170
                break;
676
170
            }
677
184k
            expr = VectorizedFnCall::create_shared(expr_node);
678
184k
            break;
679
192k
        }
680
1.71k
        case TExprNodeType::MATCH_PRED: {
681
1.71k
            expr = VMatchPredicate::create_shared(expr_node);
682
1.71k
            break;
683
192k
        }
684
151k
        case TExprNodeType::CAST_EXPR: {
685
151k
            expr = VCastExpr::create_shared(expr_node);
686
151k
            break;
687
192k
        }
688
11
        case TExprNodeType::TRY_CAST_EXPR: {
689
11
            expr = TryCastExpr::create_shared(expr_node);
690
11
            break;
691
192k
        }
692
2.24k
        case TExprNodeType::IN_PRED: {
693
2.24k
            expr = VInPredicate::create_shared(expr_node);
694
2.24k
            break;
695
192k
        }
696
294
        case TExprNodeType::CASE_EXPR: {
697
294
            if (!expr_node.__isset.case_expr) {
698
0
                return Status::InternalError("Case expression not set in thrift node");
699
0
            }
700
294
            if (expr_node.__isset.short_circuit_evaluation && expr_node.short_circuit_evaluation) {
701
55
                expr = ShortCircuitCaseExpr::create_shared(expr_node);
702
239
            } else {
703
239
                expr = VCaseExpr::create_shared(expr_node);
704
239
            }
705
294
            break;
706
294
        }
707
0
        case TExprNodeType::INFO_FUNC: {
708
0
            expr = VInfoFunc::create_shared(expr_node);
709
0
            break;
710
294
        }
711
470
        case TExprNodeType::SEARCH_EXPR: {
712
470
            expr = VSearchExpr::create_shared(expr_node);
713
470
            break;
714
294
        }
715
0
        default:
716
0
            return Status::InternalError("Unknown expr node type: {}", expr_node.node_type);
717
9.88M
        }
718
9.88M
    } catch (const Exception& e) {
719
0
        if (e.code() == ErrorCode::INTERNAL_ERROR) {
720
0
            return Status::InternalError("Create Expr failed because {}\nTExprNode={}", e.what(),
721
0
                                         apache::thrift::ThriftDebugString(expr_node));
722
0
        }
723
0
        return Status::Error<false>(e.code(), "Create Expr failed because {}", e.what());
724
0
        LOG(WARNING) << "create expr failed, TExprNode={}, reason={}"
725
0
                     << apache::thrift::ThriftDebugString(expr_node) << e.what();
726
0
    }
727
9.90M
    if (!expr->data_type()) {
728
0
        return Status::InvalidArgument("Unknown expr type: {}", expr_node.node_type);
729
0
    }
730
9.90M
    return Status::OK();
731
9.90M
}
732
// NOLINTEND(readability-function-size)
733
734
Status VExpr::create_tree_from_thrift(const std::vector<TExprNode>& nodes, int* node_idx,
735
8.01M
                                      VExprSPtr& root_expr, VExprContextSPtr& ctx) {
736
    // propagate error case
737
8.01M
    if (*node_idx >= nodes.size()) {
738
0
        return Status::InternalError("Failed to reconstruct expression tree from thrift.");
739
0
    }
740
741
    // create root expr
742
8.01M
    int root_children = nodes[*node_idx].num_children;
743
8.01M
    VExprSPtr root;
744
8.01M
    RETURN_IF_ERROR(create_expr(nodes[*node_idx], root));
745
8.01M
    DCHECK(root != nullptr);
746
8.01M
    root_expr = root;
747
8.01M
    ctx = std::make_shared<VExprContext>(root);
748
    // short path for leaf node
749
8.01M
    if (root_children <= 0) {
750
7.22M
        return Status::OK();
751
7.22M
    }
752
753
    // non-recursive traversal
754
785k
    using VExprSPtrCountPair = std::pair<VExprSPtr, int>;
755
785k
    std::stack<std::shared_ptr<VExprSPtrCountPair>> s;
756
785k
    s.emplace(std::make_shared<VExprSPtrCountPair>(root, root_children));
757
2.65M
    while (!s.empty()) {
758
        // copy the shared ptr resource to avoid dangling reference
759
1.86M
        auto parent = s.top();
760
        // Decrement or pop
761
1.86M
        if (parent->second > 1) {
762
901k
            parent->second -= 1;
763
964k
        } else {
764
964k
            s.pop();
765
964k
        }
766
767
1.86M
        DCHECK(parent->first != nullptr);
768
1.86M
        if (++*node_idx >= nodes.size()) {
769
0
            return Status::InternalError("Failed to reconstruct expression tree from thrift.");
770
0
        }
771
772
1.86M
        VExprSPtr expr;
773
1.86M
        RETURN_IF_ERROR(create_expr(nodes[*node_idx], expr));
774
1.86M
        DCHECK(expr != nullptr);
775
1.86M
        parent->first->add_child(expr);
776
        // push to stack if has children
777
1.86M
        int num_children = nodes[*node_idx].num_children;
778
1.86M
        if (num_children > 0) {
779
167k
            s.emplace(std::make_shared<VExprSPtrCountPair>(expr, num_children));
780
167k
        }
781
1.86M
    }
782
785k
    return Status::OK();
783
785k
}
784
785
7.89M
Status VExpr::create_expr_tree(const TExpr& texpr, VExprContextSPtr& ctx) {
786
7.89M
    if (texpr.nodes.empty()) {
787
9
        ctx = nullptr;
788
9
        return Status::OK();
789
9
    }
790
7.89M
    int node_idx = 0;
791
7.89M
    VExprSPtr e;
792
7.89M
    Status status = create_tree_from_thrift(texpr.nodes, &node_idx, e, ctx);
793
7.91M
    if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
794
0
        status = Status::InternalError(
795
0
                "Expression tree only partially reconstructed. Not all thrift nodes were "
796
0
                "used.");
797
0
    }
798
7.89M
    if (!status.ok()) {
799
0
        LOG(ERROR) << "Could not construct expr tree.\n"
800
0
                   << status << "\n"
801
0
                   << apache::thrift::ThriftDebugString(texpr);
802
0
    }
803
7.89M
    return status;
804
7.89M
}
805
806
1.43M
Status VExpr::create_expr_trees(const std::vector<TExpr>& texprs, VExprContextSPtrs& ctxs) {
807
1.43M
    ctxs.clear();
808
6.60M
    for (const auto& texpr : texprs) {
809
6.60M
        VExprContextSPtr ctx;
810
6.60M
        RETURN_IF_ERROR(create_expr_tree(texpr, ctx));
811
6.60M
        ctxs.push_back(ctx);
812
6.60M
    }
813
1.43M
    return Status::OK();
814
1.43M
}
815
816
Status VExpr::check_expr_output_type(const VExprContextSPtrs& ctxs,
817
333k
                                     const RowDescriptor& output_row_desc) {
818
333k
    if (ctxs.empty()) {
819
6
        return Status::OK();
820
6
    }
821
333k
    auto name_and_types = VectorizedUtils::create_name_and_data_types(output_row_desc);
822
333k
    if (ctxs.size() != name_and_types.size()) {
823
0
        return Status::InternalError(
824
0
                "output type size not match expr size {} , expected output size {} ", ctxs.size(),
825
0
                name_and_types.size());
826
0
    }
827
2.35M
    auto check_type_can_be_converted = [](DataTypePtr& from, DataTypePtr& to) -> bool {
828
2.35M
        if (to->equals(*from)) {
829
2.35M
            return true;
830
2.35M
        }
831
1.92k
        if (to->is_nullable() && !from->is_nullable()) {
832
87
            return remove_nullable(to)->equals(*from);
833
87
        }
834
1.83k
        return false;
835
1.92k
    };
836
2.68M
    for (int i = 0; i < ctxs.size(); i++) {
837
2.35M
        auto real_expr_type = get_data_type_with_default_argument(ctxs[i]->root()->data_type());
838
2.35M
        auto&& [name, expected_type] = name_and_types[i];
839
2.35M
        if (!check_type_can_be_converted(real_expr_type, expected_type)) {
840
0
            return Status::InternalError(
841
0
                    "output type not match expr type, col name {} , expected type {} , real type "
842
0
                    "{}",
843
0
                    name, expected_type->get_name(), real_expr_type->get_name());
844
0
        }
845
2.35M
    }
846
333k
    return Status::OK();
847
333k
}
848
849
Status VExpr::prepare(const VExprContextSPtrs& ctxs, RuntimeState* state,
850
2.24M
                      const RowDescriptor& row_desc) {
851
6.92M
    for (auto ctx : ctxs) {
852
6.92M
        RETURN_IF_ERROR(ctx->prepare(state, row_desc));
853
6.92M
    }
854
2.24M
    return Status::OK();
855
2.24M
}
856
857
2.26M
Status VExpr::open(const VExprContextSPtrs& ctxs, RuntimeState* state) {
858
6.92M
    for (const auto& ctx : ctxs) {
859
6.92M
        RETURN_IF_ERROR(ctx->open(state));
860
6.92M
    }
861
2.26M
    return Status::OK();
862
2.26M
}
863
864
1.77M
bool VExpr::contains_blockable_function(const VExprContextSPtrs& ctxs) {
865
2.83M
    return std::any_of(ctxs.begin(), ctxs.end(), [](const VExprContextSPtr& ctx) {
866
2.83M
        return ctx != nullptr && ctx->root() != nullptr && ctx->root()->is_blockable();
867
2.83M
    });
868
1.77M
}
869
870
Status VExpr::clone_if_not_exists(const VExprContextSPtrs& ctxs, RuntimeState* state,
871
0
                                  VExprContextSPtrs& new_ctxs) {
872
0
    if (!new_ctxs.empty()) {
873
        // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
874
0
        DCHECK_EQ(new_ctxs.size(), ctxs.size());
875
0
        for (auto& new_ctx : new_ctxs) {
876
0
            DCHECK(new_ctx->_is_clone);
877
0
        }
878
0
        return Status::OK();
879
0
    }
880
0
    new_ctxs.resize(ctxs.size());
881
0
    for (int i = 0; i < ctxs.size(); ++i) {
882
0
        RETURN_IF_ERROR(ctxs[i]->clone(state, new_ctxs[i]));
883
0
    }
884
0
    return Status::OK();
885
0
}
886
887
1.09k
std::string VExpr::debug_string() const {
888
    // TODO: implement partial debug string for member vars
889
1.09k
    std::stringstream out;
890
1.09k
    out << " type=" << _data_type->get_name();
891
892
1.09k
    if (!_children.empty()) {
893
182
        out << " children=" << debug_string(_children);
894
182
    }
895
896
1.09k
    return out.str();
897
1.09k
}
898
899
182
std::string VExpr::debug_string(const VExprSPtrs& exprs) {
900
182
    std::stringstream out;
901
182
    out << "[";
902
903
364
    for (int i = 0; i < exprs.size(); ++i) {
904
182
        out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
905
182
    }
906
907
182
    out << "]";
908
182
    return out.str();
909
182
}
910
911
0
std::string VExpr::debug_string(const VExprContextSPtrs& ctxs) {
912
0
    VExprSPtrs exprs;
913
0
    for (const auto& ctx : ctxs) {
914
0
        exprs.push_back(ctx->root());
915
0
    }
916
0
    return debug_string(exprs);
917
0
}
918
919
7.91M
bool VExpr::is_constant() const {
920
7.91M
    return std::all_of(_children.begin(), _children.end(),
921
7.91M
                       [](const VExprSPtr& expr) { return expr->is_constant(); });
922
7.91M
}
923
924
Status VExpr::get_const_col(VExprContext* context,
925
12.5M
                            std::shared_ptr<ColumnPtrWrapper>* column_wrapper) {
926
12.5M
    if (!is_constant()) {
927
8.79M
        return Status::OK();
928
8.79M
    }
929
930
3.79M
    if (_constant_col != nullptr && column_wrapper == nullptr) {
931
0
        return Status::OK();
932
3.79M
    } else if (_constant_col != nullptr) {
933
1.39M
        *column_wrapper = _constant_col;
934
1.39M
        return Status::OK();
935
1.39M
    }
936
937
2.40M
    ColumnPtr result;
938
2.40M
    RETURN_IF_ERROR(execute_column(context, nullptr, nullptr, 1, result));
939
2.40M
    _constant_col = std::make_shared<ColumnPtrWrapper>(result);
940
2.40M
    if (column_wrapper != nullptr) {
941
0
        *column_wrapper = _constant_col;
942
0
    }
943
944
2.40M
    return Status::OK();
945
2.40M
}
946
947
1.03M
void VExpr::register_function_context(RuntimeState* state, VExprContext* context) {
948
1.03M
    std::vector<DataTypePtr> arg_types;
949
1.87M
    for (auto& i : _children) {
950
1.87M
        arg_types.push_back(i->data_type());
951
1.87M
    }
952
953
1.03M
    _fn_context_index = context->register_function_context(state, _data_type, arg_types);
954
1.03M
}
955
956
Status VExpr::init_function_context(RuntimeState* state, VExprContext* context,
957
                                    FunctionContext::FunctionStateScope scope,
958
2.65M
                                    const FunctionBasePtr& function) const {
959
2.65M
    FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
960
2.65M
    if (scope == FunctionContext::FRAGMENT_LOCAL) {
961
1.03M
        std::vector<std::shared_ptr<ColumnPtrWrapper>> constant_cols;
962
1.87M
        for (auto c : _children) {
963
1.87M
            std::shared_ptr<ColumnPtrWrapper> const_col;
964
1.87M
            RETURN_IF_ERROR(c->get_const_col(context, &const_col));
965
1.87M
            constant_cols.push_back(const_col);
966
1.87M
        }
967
1.03M
        fn_ctx->set_constant_cols(constant_cols);
968
1.03M
    }
969
970
2.65M
    if (scope == FunctionContext::FRAGMENT_LOCAL) {
971
1.03M
        RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
972
1.03M
    }
973
2.65M
    RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::THREAD_LOCAL));
974
2.65M
    return Status::OK();
975
2.65M
}
976
977
void VExpr::close_function_context(VExprContext* context, FunctionContext::FunctionStateScope scope,
978
2.66M
                                   const FunctionBasePtr& function) const {
979
2.66M
    if (_fn_context_index != -1) {
980
2.66M
        FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
981
        // `close_function_context` is called in VExprContext's destructor so do not throw exceptions here.
982
2.66M
        static_cast<void>(function->close(fn_ctx, FunctionContext::THREAD_LOCAL));
983
2.66M
        if (scope == FunctionContext::FRAGMENT_LOCAL) {
984
1.03M
            static_cast<void>(function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
985
1.03M
        }
986
2.66M
    }
987
2.66M
}
988
989
0
Status VExpr::check_constant(const Block& block, ColumnNumbers arguments) const {
990
0
    if (is_constant() && !VectorizedUtils::all_arguments_are_constant(block, arguments)) {
991
0
        return Status::InternalError("const check failed, expr={}", debug_string());
992
0
    }
993
0
    return Status::OK();
994
0
}
995
996
615k
uint64_t VExpr::get_digest(uint64_t seed) const {
997
615k
    auto digest = seed;
998
1.21M
    for (auto child : _children) {
999
1.21M
        digest = child->get_digest(digest);
1000
1.21M
        if (digest == 0) {
1001
543
            return 0;
1002
543
        }
1003
1.21M
    }
1004
1005
615k
    auto& fn_name = _fn.name.function_name;
1006
615k
    if (!fn_name.empty()) {
1007
610k
        digest = HashUtil::hash64(fn_name.c_str(), fn_name.size(), digest);
1008
610k
    } else {
1009
4.64k
        digest = HashUtil::hash64((const char*)&_node_type, sizeof(_node_type), digest);
1010
4.64k
        digest = HashUtil::hash64((const char*)&_opcode, sizeof(_opcode), digest);
1011
4.64k
    }
1012
615k
    return digest;
1013
615k
}
1014
1015
89.4k
ColumnPtr VExpr::get_result_from_const(size_t count) const {
1016
89.4k
    return ColumnConst::create(_constant_col->column_ptr, count);
1017
89.4k
}
1018
1019
Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function,
1020
19.2k
                                       uint32_t segment_num_rows) {
1021
    // Pre-allocate vectors based on an estimated or known size
1022
19.2k
    std::vector<segment_v2::IndexIterator*> iterators;
1023
19.2k
    std::vector<IndexFieldNameAndTypePair> data_type_with_names;
1024
19.2k
    std::vector<int> column_ids;
1025
19.2k
    ColumnsWithTypeAndName arguments;
1026
19.2k
    VExprSPtrs children_exprs;
1027
1028
    // Reserve space to avoid multiple reallocations
1029
19.2k
    const size_t estimated_size = get_num_children();
1030
19.2k
    iterators.reserve(estimated_size);
1031
19.2k
    data_type_with_names.reserve(estimated_size);
1032
19.2k
    column_ids.reserve(estimated_size);
1033
19.2k
    children_exprs.reserve(estimated_size);
1034
1035
19.2k
    auto index_context = context->get_index_context();
1036
1037
    // if child is cast expr, we need to ensure target data type is the same with storage data type.
1038
    // or they are all string type
1039
    // and if data type is array, we need to get the nested data type to ensure that.
1040
39.1k
    for (const auto& child : children()) {
1041
39.1k
        if (child->node_type() == TExprNodeType::CAST_EXPR) {
1042
4.77k
            auto* cast_expr = assert_cast<VCastExpr*>(child.get());
1043
4.77k
            DCHECK_EQ(cast_expr->get_num_children(), 1);
1044
4.77k
            if (cast_expr->get_child(0)->is_slot_ref()) {
1045
4.64k
                auto* column_slot_ref = assert_cast<VSlotRef*>(cast_expr->get_child(0).get());
1046
4.64k
                auto column_id = column_slot_ref->column_id();
1047
4.64k
                const auto* storage_name_type =
1048
4.64k
                        context->get_index_context()->get_storage_name_and_type_by_column_id(
1049
4.64k
                                column_id);
1050
4.64k
                auto storage_type = remove_nullable(storage_name_type->second);
1051
4.64k
                auto target_type = remove_nullable(cast_expr->get_target_type());
1052
4.64k
                auto origin_primitive_type = storage_type->get_primitive_type();
1053
4.64k
                auto target_primitive_type = target_type->get_primitive_type();
1054
4.64k
                if (is_complex_type(storage_type->get_primitive_type())) {
1055
489
                    if (storage_type->get_primitive_type() == TYPE_ARRAY &&
1056
490
                        target_type->get_primitive_type() == TYPE_ARRAY) {
1057
480
                        auto nested_storage_type =
1058
480
                                (assert_cast<const DataTypeArray*>(storage_type.get()))
1059
480
                                        ->get_nested_type();
1060
480
                        origin_primitive_type = nested_storage_type->get_primitive_type();
1061
480
                        auto nested_target_type =
1062
480
                                (assert_cast<const DataTypeArray*>(target_type.get()))
1063
480
                                        ->get_nested_type();
1064
480
                        target_primitive_type = nested_target_type->get_primitive_type();
1065
480
                    } else {
1066
9
                        continue;
1067
9
                    }
1068
489
                }
1069
4.63k
                if (origin_primitive_type != TYPE_VARIANT &&
1070
4.63k
                    (storage_type->equals(*target_type) ||
1071
3.72k
                     (is_string_type(target_primitive_type) &&
1072
2.67k
                      is_string_type(origin_primitive_type)))) {
1073
1.07k
                    children_exprs.emplace_back(expr_without_cast(child));
1074
1.07k
                }
1075
4.63k
            } else {
1076
135
                return Status::OK(); // for example: cast("abc") as ipv4 case
1077
135
            }
1078
34.4k
        } else {
1079
34.4k
            children_exprs.emplace_back(child);
1080
34.4k
        }
1081
39.1k
    }
1082
1083
19.0k
    if (children_exprs.empty()) {
1084
582
        return Status::OK(); // Early exit if no children to process
1085
582
    }
1086
1087
32.7k
    for (const auto& child : children_exprs) {
1088
32.7k
        if (child->is_slot_ref()) {
1089
14.0k
            auto* column_slot_ref = assert_cast<VSlotRef*>(child.get());
1090
14.0k
            auto column_id = column_slot_ref->column_id();
1091
14.0k
            auto* iter = context->get_index_context()->get_inverted_index_iterator_by_column_id(
1092
14.0k
                    column_id);
1093
            //column does not have inverted index
1094
14.0k
            if (iter == nullptr) {
1095
4.37k
                continue;
1096
4.37k
            }
1097
9.65k
            const auto* storage_name_type =
1098
9.65k
                    context->get_index_context()->get_storage_name_and_type_by_column_id(column_id);
1099
9.65k
            if (storage_name_type == nullptr) {
1100
0
                auto err_msg = fmt::format(
1101
0
                        "storage_name_type cannot be found for column {} while in {} "
1102
0
                        "evaluate_inverted_index",
1103
0
                        column_id, expr_name());
1104
0
                LOG(ERROR) << err_msg;
1105
0
                return Status::InternalError(err_msg);
1106
0
            }
1107
9.65k
            iterators.emplace_back(iter);
1108
9.65k
            data_type_with_names.emplace_back(*storage_name_type);
1109
9.65k
            column_ids.emplace_back(column_id);
1110
18.7k
        } else if (child->is_literal()) {
1111
15.9k
            auto* column_literal = assert_cast<VLiteral*>(child.get());
1112
15.9k
            arguments.emplace_back(column_literal->get_column_ptr(),
1113
15.9k
                                   column_literal->get_data_type(), column_literal->expr_name());
1114
15.9k
        } else if (child->can_push_down_to_index()) {
1115
0
            RETURN_IF_ERROR(child->evaluate_inverted_index(context, segment_num_rows));
1116
2.76k
        } else {
1117
2.76k
            return Status::OK(); // others cases
1118
2.76k
        }
1119
32.7k
    }
1120
1121
    // is null or is not null has no arguments
1122
15.7k
    if (iterators.empty() || (arguments.empty() && !(function->get_name() == "is_not_null_pred" ||
1123
6.55k
                                                     function->get_name() == "is_null_pred"))) {
1124
6.55k
        return Status::OK(); // Nothing to evaluate or no literals to compare against
1125
6.55k
    }
1126
1127
9.18k
    const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr;
1128
9.35k
    if (auto index_ctx = context->get_index_context(); index_ctx != nullptr) {
1129
9.35k
        analyzer_ctx = index_ctx->get_analyzer_ctx_for_expr(this);
1130
9.35k
    }
1131
1132
9.18k
    auto result_bitmap = segment_v2::InvertedIndexResultBitmap();
1133
    // Pass analyzer_key to function (used by match predicates for multi-analyzer index selection)
1134
9.18k
    auto res = function->evaluate_inverted_index(arguments, data_type_with_names, iterators,
1135
9.18k
                                                 segment_num_rows, analyzer_ctx, result_bitmap);
1136
9.18k
    if (!res.ok()) {
1137
348
        return res;
1138
348
    }
1139
8.83k
    if (!result_bitmap.is_empty()) {
1140
7.41k
        index_context->set_index_result_for_expr(this, result_bitmap);
1141
7.55k
        for (int column_id : column_ids) {
1142
7.55k
            index_context->set_true_for_index_status(this, column_id);
1143
7.55k
        }
1144
7.41k
    }
1145
8.83k
    return Status::OK();
1146
9.18k
}
1147
1148
0
size_t VExpr::estimate_memory(const size_t rows) {
1149
0
    if (is_const_and_have_executed()) {
1150
0
        return 0;
1151
0
    }
1152
1153
0
    size_t estimate_size = 0;
1154
0
    for (auto& child : _children) {
1155
0
        estimate_size += child->estimate_memory(rows);
1156
0
    }
1157
1158
0
    if (_data_type->have_maximum_size_of_value()) {
1159
0
        estimate_size += rows * _data_type->get_size_of_value_in_memory();
1160
0
    } else {
1161
0
        estimate_size += rows * 64; /// TODO: need a more reasonable value
1162
0
    }
1163
0
    return estimate_size;
1164
0
}
1165
1166
Status VExpr::execute_column(VExprContext* context, const Block* block, const Selector* selector,
1167
8.42M
                             size_t count, ColumnPtr& result_column) const {
1168
8.42M
    RETURN_IF_ERROR(execute_column_impl(context, block, selector, count, result_column));
1169
8.42M
    if (result_column->size() != count) {
1170
1
        return Status::InternalError("Expr {} return column size {} not equal to expected size {}",
1171
1
                                     expr_name(), result_column->size(), count);
1172
1
    }
1173
8.42M
    DCHECK(selector == nullptr || selector->size() == count);
1174
    // Validate type match. ColumnNothing is exempt (used as a placeholder in tests/stubs).
1175
8.42M
    if (!check_and_get_column<ColumnNothing>(result_column.get())) {
1176
8.42M
        auto result_type = execute_type(block);
1177
8.42M
        if (result_type != nullptr) {
1178
8.42M
            Status st = result_type->check_column(*result_column);
1179
8.42M
            if (!st.ok()) {
1180
                // Nullable(T) may legitimately produce a non-nullable T column when all rows are
1181
                // non-null (use_default_implementation_for_nulls optimization). Allow this.
1182
5.05k
                const auto* nullable_type =
1183
5.05k
                        check_and_get_data_type<DataTypeNullable>(result_type.get());
1184
5.05k
                if (nullable_type && !check_and_get_column<ColumnNullable>(result_column.get())) {
1185
5.05k
                    st = nullable_type->get_nested_type()->check_column(*result_column);
1186
5.05k
                }
1187
5.05k
            }
1188
8.42M
            if (!st.ok()) {
1189
1
                return Status::InternalError(
1190
1
                        "Expr {} return column type mismatch: declared={}, actual={}", expr_name(),
1191
1
                        result_type->get_name(), result_column->get_name());
1192
1
            }
1193
8.42M
        }
1194
8.42M
    }
1195
8.42M
    return Status::OK();
1196
8.42M
}
1197
1198
bool VExpr::fast_execute(VExprContext* context, const Selector* selector, size_t count,
1199
698k
                         ColumnPtr& result_column) const {
1200
698k
    if (context->get_index_context() &&
1201
698k
        context->get_index_context()->get_index_result_column().contains(this)) {
1202
        // prepare a column to save result
1203
775
        result_column = filter_column_with_selector(
1204
775
                context->get_index_context()->get_index_result_column()[this], selector, count);
1205
775
        if (_data_type->is_nullable()) {
1206
665
            result_column = make_nullable(result_column);
1207
665
        }
1208
775
        return true;
1209
775
    }
1210
697k
    return false;
1211
698k
}
1212
1213
0
bool VExpr::equals(const VExpr& other) {
1214
0
    return false;
1215
0
}
1216
1217
Status VExpr::evaluate_ann_range_search(
1218
        const segment_v2::AnnRangeSearchRuntime& runtime,
1219
        const std::vector<std::unique_ptr<segment_v2::IndexIterator>>& index_iterators,
1220
        const std::vector<ColumnId>& idx_to_cid,
1221
        const std::vector<std::unique_ptr<segment_v2::ColumnIterator>>& column_iterators,
1222
        size_t rows_of_segment, roaring::Roaring& row_bitmap, AnnIndexStats& ann_index_stats,
1223
6.51k
        bool enable_result_cache, AnnRangeSearchEvaluationResult& result) {
1224
6.51k
    result = {};
1225
6.51k
    return Status::OK();
1226
6.51k
}
1227
1228
void VExpr::prepare_ann_range_search(const doris::VectorSearchUserParams& params,
1229
                                     segment_v2::AnnRangeSearchRuntime& range_search_runtime,
1230
19.5k
                                     bool& suitable_for_ann_index) {
1231
19.5k
    if (!suitable_for_ann_index) {
1232
0
        return;
1233
0
    }
1234
19.5k
    for (auto& child : _children) {
1235
12.2k
        child->prepare_ann_range_search(params, range_search_runtime, suitable_for_ann_index);
1236
12.2k
        if (!suitable_for_ann_index) {
1237
808
            return;
1238
808
        }
1239
12.2k
    }
1240
19.5k
}
1241
1242
Status VExpr::execute_filter(VExprContext* context, const Block* block,
1243
                             uint8_t* __restrict result_filter_data, size_t rows, bool accept_null,
1244
299k
                             bool* can_filter_all) const {
1245
299k
    ColumnPtr filter_column;
1246
299k
    RETURN_IF_ERROR(execute_column(context, block, nullptr, rows, filter_column));
1247
299k
    if (const auto* const_column = check_and_get_column<ColumnConst>(*filter_column)) {
1248
        // const(nullable) or const(bool)
1249
58.9k
        const bool result = accept_null
1250
58.9k
                                    ? (const_column->is_null_at(0) || const_column->get_bool(0))
1251
58.9k
                                    : (!const_column->is_null_at(0) && const_column->get_bool(0));
1252
58.9k
        if (!result) {
1253
            // filter all
1254
10.3k
            *can_filter_all = true;
1255
10.3k
            memset(result_filter_data, 0, rows);
1256
10.3k
            return Status::OK();
1257
10.3k
        }
1258
240k
    } else if (const auto* nullable_column = check_and_get_column<ColumnNullable>(*filter_column)) {
1259
        // nullable(bool)
1260
182k
        const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr();
1261
182k
        const IColumn::Filter& filter = assert_cast<const ColumnUInt8&>(*nested_column).get_data();
1262
182k
        const auto* __restrict filter_data = filter.data();
1263
182k
        const auto* __restrict null_map_data = nullable_column->get_null_map_data().data();
1264
1265
182k
        if (accept_null) {
1266
2
            for (size_t i = 0; i < rows; ++i) {
1267
1
                result_filter_data[i] &= (null_map_data[i]) || filter_data[i];
1268
1
            }
1269
182k
        } else {
1270
107M
            for (size_t i = 0; i < rows; ++i) {
1271
107M
                result_filter_data[i] &= (!null_map_data[i]) & filter_data[i];
1272
107M
            }
1273
182k
        }
1274
1275
182k
        if (!simd::contain_one(result_filter_data, rows)) {
1276
38.4k
            *can_filter_all = true;
1277
38.4k
            return Status::OK();
1278
38.4k
        }
1279
182k
    } else {
1280
        // bool
1281
58.3k
        const IColumn::Filter& filter = assert_cast<const ColumnUInt8&>(*filter_column).get_data();
1282
58.3k
        const auto* __restrict filter_data = filter.data();
1283
1284
67.1M
        for (size_t i = 0; i < rows; ++i) {
1285
67.1M
            result_filter_data[i] &= filter_data[i];
1286
67.1M
        }
1287
1288
58.3k
        if (!simd::contain_one(result_filter_data, rows)) {
1289
6.58k
            *can_filter_all = true;
1290
6.58k
            return Status::OK();
1291
6.58k
        }
1292
58.3k
    }
1293
1294
244k
    return Status::OK();
1295
299k
}
1296
1297
} // namespace doris