/root/doris/be/src/exprs/vmatch_predicate.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "exprs/vmatch_predicate.h" |
19 | | |
20 | | #include <cstdint> |
21 | | |
22 | | #ifdef __clang__ |
23 | | #pragma clang diagnostic push |
24 | | #pragma clang diagnostic ignored "-Wshadow-field" |
25 | | #endif |
26 | | |
27 | | #include <fmt/format.h> |
28 | | #include <fmt/ranges.h> // IWYU pragma: keep |
29 | | #include <gen_cpp/Exprs_types.h> |
30 | | #include <glog/logging.h> |
31 | | |
32 | | #include <memory> |
33 | | #include <string> |
34 | | #include <string_view> |
35 | | #include <type_traits> |
36 | | #include <vector> |
37 | | |
38 | | #include "common/status.h" |
39 | | #include "core/block/block.h" |
40 | | #include "core/block/column_numbers.h" |
41 | | #include "core/block/column_with_type_and_name.h" |
42 | | #include "exprs/function/match.h" |
43 | | #include "exprs/function/simple_function_factory.h" |
44 | | #include "exprs/vexpr_context.h" |
45 | | #include "exprs/vslot_ref.h" |
46 | | #include "runtime/runtime_state.h" |
47 | | #include "storage/index/inverted/analyzer/analyzer.h" |
48 | | #include "storage/index/inverted/inverted_index_reader.h" |
49 | | |
50 | | namespace doris { |
51 | | class RowDescriptor; |
52 | | class RuntimeState; |
53 | | } // namespace doris |
54 | | |
55 | | namespace doris { |
56 | | #include "common/compile_check_begin.h" |
57 | | |
58 | | using namespace doris::segment_v2; |
59 | | |
60 | 24 | VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) { |
61 | 24 | const auto resolved = AnalyzerConfigParser::parse( |
62 | 24 | node.match_predicate.analyzer_name, node.match_predicate.parser_type, |
63 | 24 | node.match_predicate.parser_mode, node.match_predicate.parser_lowercase, |
64 | 24 | node.match_predicate.char_filter_map); |
65 | | |
66 | 24 | InvertedIndexAnalyzerConfig config; |
67 | 24 | config.analyzer_name = resolved.provider_name; |
68 | 24 | config.parser_type = resolved.parser_type; |
69 | 24 | config.parser_mode = node.match_predicate.parser_mode; |
70 | 24 | config.char_filter_map = node.match_predicate.char_filter_map; |
71 | 24 | if (node.match_predicate.parser_lowercase) { |
72 | 22 | config.lower_case = INVERTED_INDEX_PARSER_TRUE; |
73 | 22 | } else { |
74 | 2 | config.lower_case = INVERTED_INDEX_PARSER_FALSE; |
75 | 2 | } |
76 | 24 | DBUG_EXECUTE_IF("inverted_index_parser.get_parser_lowercase_from_properties", |
77 | 24 | { config.lower_case = ""; }) |
78 | 24 | config.stop_words = node.match_predicate.parser_stopwords; |
79 | | |
80 | 24 | _analyzer_ctx = std::make_shared<InvertedIndexAnalyzerCtx>(); |
81 | 24 | _analyzer_ctx->analyzer_key = resolved.analyzer_key; |
82 | 24 | _analyzer_ctx->analyzer_name = resolved.provider_name; |
83 | 24 | _analyzer_ctx->parser_type = resolved.parser_type; |
84 | | |
85 | 24 | if (_analyzer_ctx->requires_analysis()) { |
86 | 24 | std::string bound_name; |
87 | 24 | std::string legacy_name; |
88 | 24 | _analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( |
89 | 24 | &config, &bound_name, &legacy_name); |
90 | 24 | _analyzer = _analyzer_provider->get_analyzer(); |
91 | 24 | if (!legacy_name.empty() && legacy_name != bound_name) { |
92 | 11 | _analyzer_ctx->legacy_analyzer_key = |
93 | 11 | AnalyzerConfigParser::parse(legacy_name, node.match_predicate.parser_type, |
94 | 11 | node.match_predicate.parser_mode, |
95 | 11 | node.match_predicate.parser_lowercase, |
96 | 11 | node.match_predicate.char_filter_map) |
97 | 11 | .analyzer_key; |
98 | 11 | } |
99 | 24 | if (bound_name != resolved.provider_name) { |
100 | | // Reader selection and query tokenization must use the same policy binding. |
101 | 7 | _analyzer_ctx->analyzer_key = |
102 | 7 | AnalyzerConfigParser::parse(bound_name, node.match_predicate.parser_type, |
103 | 7 | node.match_predicate.parser_mode, |
104 | 7 | node.match_predicate.parser_lowercase, |
105 | 7 | node.match_predicate.char_filter_map) |
106 | 7 | .analyzer_key; |
107 | 7 | _analyzer_ctx->analyzer_name = std::move(bound_name); |
108 | 7 | } |
109 | 24 | } |
110 | | |
111 | 24 | _analyzer_ctx->char_filter_map = std::move(config.char_filter_map); |
112 | 24 | _analyzer_ctx->analyzer = _analyzer; |
113 | 24 | _analyzer_ctx->analyzer_provider = _analyzer_provider; |
114 | 24 | } |
115 | | |
116 | 24 | VMatchPredicate::~VMatchPredicate() = default; |
117 | | |
118 | | Status VMatchPredicate::prepare(RuntimeState* state, const RowDescriptor& desc, |
119 | 0 | VExprContext* context) { |
120 | 0 | RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); |
121 | |
|
122 | 0 | ColumnsWithTypeAndName argument_template; |
123 | 0 | argument_template.reserve(_children.size()); |
124 | 0 | std::vector<std::string_view> child_expr_name; |
125 | 0 | for (const auto& child : _children) { |
126 | 0 | argument_template.emplace_back(nullptr, child->data_type(), child->expr_name()); |
127 | 0 | child_expr_name.emplace_back(child->expr_name()); |
128 | 0 | } |
129 | |
|
130 | 0 | _function = SimpleFunctionFactory::instance().get_function(_fn.name.function_name, |
131 | 0 | argument_template, _data_type, {}); |
132 | 0 | if (_function == nullptr) { |
133 | 0 | std::string type_str; |
134 | 0 | for (const auto& arg : argument_template) { |
135 | 0 | type_str = type_str + " " + arg.type->get_name(); |
136 | 0 | } |
137 | 0 | return Status::NotSupported( |
138 | 0 | "Function {} is not implemented, input param type is {}, " |
139 | 0 | "and return type is {}.", |
140 | 0 | _fn.name.function_name, type_str, _data_type->get_name()); |
141 | 0 | } |
142 | | |
143 | 0 | VExpr::register_function_context(state, context); |
144 | 0 | _expr_name = fmt::format("{}({})", _fn.name.function_name, child_expr_name); |
145 | 0 | _function_name = _fn.name.function_name; |
146 | 0 | _prepare_finished = true; |
147 | 0 | return Status::OK(); |
148 | 0 | } |
149 | | |
150 | | Status VMatchPredicate::open(RuntimeState* state, VExprContext* context, |
151 | 0 | FunctionContext::FunctionStateScope scope) { |
152 | 0 | DCHECK(_prepare_finished); |
153 | 0 | for (auto& i : _children) { |
154 | 0 | RETURN_IF_ERROR(i->open(state, context, scope)); |
155 | 0 | } |
156 | 0 | RETURN_IF_ERROR(VExpr::init_function_context(state, context, scope, _function)); |
157 | 0 | if (scope == FunctionContext::THREAD_LOCAL || scope == FunctionContext::FRAGMENT_LOCAL) { |
158 | 0 | context->fn_context(_fn_context_index)->set_function_state(scope, _analyzer_ctx); |
159 | 0 | } |
160 | 0 | if (scope == FunctionContext::FRAGMENT_LOCAL) { |
161 | 0 | RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr)); |
162 | 0 | } |
163 | 0 | _open_finished = true; |
164 | 0 | return Status::OK(); |
165 | 0 | } |
166 | | |
167 | 0 | void VMatchPredicate::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { |
168 | 0 | VExpr::close_function_context(context, scope, _function); |
169 | 0 | VExpr::close(context, scope); |
170 | 0 | } |
171 | | |
172 | 0 | Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { |
173 | 0 | DCHECK_EQ(get_num_children(), 2); |
174 | 0 | if (context != nullptr && context->get_index_context() != nullptr && _analyzer_ctx != nullptr) { |
175 | 0 | context->get_index_context()->set_analyzer_ctx_for_expr(this, _analyzer_ctx); |
176 | 0 | } |
177 | 0 | return _evaluate_inverted_index(context, _function, segment_num_rows); |
178 | 0 | } |
179 | | |
180 | 17 | const std::string& VMatchPredicate::get_analyzer_key() const { |
181 | 17 | return _analyzer_ctx->analyzer_key; |
182 | 17 | } |
183 | | |
184 | | Status VMatchPredicate::execute_column(VExprContext* context, const Block* block, |
185 | | Selector* selector, size_t count, |
186 | 0 | ColumnPtr& result_column) const { |
187 | 0 | DCHECK(_open_finished || block == nullptr); |
188 | 0 | if (fast_execute(context, selector, count, result_column)) { |
189 | 0 | return Status::OK(); |
190 | 0 | } |
191 | 0 | DBUG_EXECUTE_IF("VMatchPredicate.execute", { |
192 | 0 | return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( |
193 | 0 | "{} not support slow path, hit debug point.", _expr_name); |
194 | 0 | }); |
195 | 0 | DBUG_EXECUTE_IF("VMatchPredicate.must_in_slow_path", { |
196 | 0 | auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default<std::string>( |
197 | 0 | "VMatchPredicate.must_in_slow_path", "column_name", ""); |
198 | |
|
199 | 0 | std::vector<std::string> column_names; |
200 | 0 | boost::split(column_names, debug_col_name, boost::algorithm::is_any_of(",")); |
201 | |
|
202 | 0 | auto* column_slot_ref = assert_cast<VSlotRef*>(get_child(0).get()); |
203 | 0 | std::string column_name = column_slot_ref->expr_name(); |
204 | 0 | auto it = std::ranges::find(column_names, column_name); |
205 | 0 | if (it == column_names.end()) { |
206 | 0 | return Status::Error<ErrorCode::INTERNAL_ERROR>( |
207 | 0 | "column {} should in slow path while VMatchPredicate::execute.", column_name); |
208 | 0 | } |
209 | 0 | }) |
210 | 0 | ColumnNumbers arguments(_children.size()); |
211 | 0 | Block temp_block; |
212 | 0 | for (size_t i = 0; i < _children.size(); ++i) { |
213 | 0 | ColumnPtr arg_column; |
214 | 0 | RETURN_IF_ERROR(_children[i]->execute_column(context, block, selector, count, arg_column)); |
215 | 0 | auto arg_type = _children[i]->execute_type(block); |
216 | 0 | temp_block.insert({arg_column, arg_type, _children[i]->expr_name()}); |
217 | 0 | arguments[i] = static_cast<uint32_t>(i); |
218 | 0 | } |
219 | 0 | uint32_t num_columns_without_result = temp_block.columns(); |
220 | | // prepare a column to save result |
221 | 0 | temp_block.insert({nullptr, _data_type, _expr_name}); |
222 | |
|
223 | 0 | RETURN_IF_ERROR(_function->execute(context->fn_context(_fn_context_index), temp_block, |
224 | 0 | arguments, num_columns_without_result, temp_block.rows())); |
225 | 0 | result_column = temp_block.get_by_position(num_columns_without_result).column; |
226 | 0 | DCHECK_EQ(result_column->size(), count); |
227 | 0 | return Status::OK(); |
228 | 0 | } |
229 | | |
230 | 0 | const std::string& VMatchPredicate::expr_name() const { |
231 | 0 | return _expr_name; |
232 | 0 | } |
233 | | |
234 | 0 | const std::string& VMatchPredicate::function_name() const { |
235 | 0 | return _function_name; |
236 | 0 | } |
237 | | |
238 | 0 | std::string VMatchPredicate::debug_string() const { |
239 | 0 | std::stringstream out; |
240 | 0 | out << "MatchPredicate(" << children()[0]->debug_string() << ",["; |
241 | 0 | uint16_t num_children = get_num_children(); |
242 | |
|
243 | 0 | for (uint16_t i = 1; i < num_children; ++i) { |
244 | 0 | out << (i == 1 ? "" : " ") << children()[i]->debug_string(); |
245 | 0 | } |
246 | |
|
247 | 0 | out << "])"; |
248 | 0 | return out.str(); |
249 | 0 | } |
250 | | |
251 | | #include "common/compile_check_end.h" |
252 | | } // namespace doris |