Coverage Report

Created: 2026-09-29 03:31

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exprs/function/variant_inverted_index_search.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <CLucene.h>
21
#include <gen_cpp/Exprs_types.h>
22
23
#include <cstdint>
24
#include <functional>
25
#include <map>
26
#include <memory>
27
#include <roaring/roaring.hh>
28
#include <string>
29
#include <unordered_map>
30
#include <unordered_set>
31
#include <utility>
32
#include <vector>
33
34
#include "common/status.h"
35
#include "core/block/columns_with_type_and_name.h"
36
#include "core/data_type/data_type.h"
37
#include "storage/index/index_query_context.h"
38
#include "storage/index/inverted/inverted_index_cache.h"
39
#include "storage/index/inverted/inverted_index_iterator.h"
40
#include "storage/index/inverted/inverted_index_reader.h"
41
#include "storage/index/inverted/query_v2/query.h"
42
#include "storage/index/inverted/query_v2/weight.h"
43
#include "storage/olap_common.h"
44
#include "storage/segment/column_reader.h"
45
46
namespace doris::segment_v2::inverted_index::query_v2 {
47
class Query;
48
}
49
50
namespace doris::segment_v2 {
51
class NestedGroupReadProvider;
52
struct NestedGroupReader;
53
class VariantColumnReader;
54
} // namespace doris::segment_v2
55
56
namespace doris {
57
58
using namespace doris::segment_v2;
59
60
class FunctionSearch;
61
class IndexExecContext;
62
63
using SearchLeafQueryMapper = std::function<Status(
64
        const std::string&, std::shared_ptr<segment_v2::inverted_index::query_v2::Query>*)>;
65
66
enum class SearchFieldBindingState {
67
    BOUND,
68
    MISSING_IN_SEGMENT,
69
};
70
71
enum class SearchFieldExecutionMode {
72
    UNBOUND,
73
    CLUCENE,
74
    DIRECT_INDEX,
75
    SNII_NATIVE,
76
};
77
78
struct FieldReaderBinding {
79
    std::string logical_field_name;
80
    std::string stored_field_name;
81
    std::wstring stored_field_wstr;
82
    DataTypePtr column_type;
83
    InvertedIndexQueryType query_type;
84
    InvertedIndexReaderPtr inverted_reader;
85
    std::shared_ptr<lucene::index::IndexReader> lucene_reader;
86
    std::map<std::string, std::string> index_properties;
87
    std::string binding_key;
88
    std::string analyzer_key;
89
    InvertedIndexAnalyzerCtxSPtr analyzer_context;
90
    SearchFieldBindingState state = SearchFieldBindingState::MISSING_IN_SEGMENT;
91
    SearchFieldExecutionMode execution_mode = SearchFieldExecutionMode::UNBOUND;
92
93
80
    bool is_bound() const {
94
80
        return state == SearchFieldBindingState::BOUND || inverted_reader != nullptr ||
95
80
               lucene_reader != nullptr;
96
80
    }
97
8
    bool use_direct_index_reader() const {
98
8
        return is_bound() && execution_mode == SearchFieldExecutionMode::DIRECT_INDEX;
99
8
    }
100
45
    bool use_snii_native_reader() const {
101
45
        return is_bound() && execution_mode == SearchFieldExecutionMode::SNII_NATIVE;
102
45
    }
103
};
104
105
// Build the analyzer context a SEARCH binding executes with, converting a failure to build the
106
// analyzer provider into a Status instead of letting the exception escape a Status-returning caller.
107
Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
108
        const std::map<std::string, std::string>& properties, const std::string& analyzer_key);
109
110
class FieldReaderResolver {
111
public:
112
    FieldReaderResolver(
113
            const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
114
            const std::unordered_map<std::string, IndexIterator*>& iterators,
115
            std::shared_ptr<IndexQueryContext> context,
116
            const std::vector<TSearchFieldBinding>& field_bindings = {});
117
118
    Status resolve(const std::string& field_name, InvertedIndexQueryType query_type,
119
                   FieldReaderBinding* binding);
120
121
    Status resolve_with_analyzer_context(const std::string& field_name,
122
                                         InvertedIndexQueryType query_type,
123
                                         FieldReaderBinding* binding);
124
125
62
    bool is_variant_subcolumn(const std::string& field_name) const {
126
62
        return _variant_subcolumn_fields.count(field_name) > 0;
127
62
    }
128
129
11
    const std::vector<std::shared_ptr<lucene::index::IndexReader>>& readers() const {
130
11
        return _readers;
131
11
    }
132
133
    const std::unordered_map<std::string, std::shared_ptr<lucene::index::IndexReader>>&
134
10
    reader_bindings() const {
135
10
        return _binding_readers;
136
10
    }
137
138
    const std::unordered_map<std::wstring, std::shared_ptr<lucene::index::IndexReader>>&
139
10
    field_readers() const {
140
10
        return _field_readers;
141
10
    }
142
143
14
    const std::unordered_map<std::string, FieldReaderBinding>& binding_cache() const {
144
14
        return _cache;
145
14
    }
146
147
2
    IndexIterator* get_iterator(const std::string& field_name) const {
148
2
        auto it = _iterators.find(field_name);
149
2
        return (it != _iterators.end()) ? it->second : nullptr;
150
2
    }
151
152
2
    void set_leaf_query_mapper(SearchLeafQueryMapper mapper) {
153
2
        _leaf_query_mapper = std::move(mapper);
154
2
    }
155
156
    Status map_leaf_query(
157
            const std::string& field_name,
158
24
            std::shared_ptr<segment_v2::inverted_index::query_v2::Query>* query) const {
159
24
        if (!_leaf_query_mapper || query == nullptr || *query == nullptr) {
160
22
            return Status::OK();
161
22
        }
162
2
        return _leaf_query_mapper(field_name, query);
163
24
    }
164
165
private:
166
    std::string binding_key_for(const std::string& stored_field_name,
167
61
                                InvertedIndexQueryType query_type) const {
168
61
        return stored_field_name + "#" + std::to_string(static_cast<int>(query_type));
169
61
    }
170
171
    const std::unordered_map<std::string, IndexFieldNameAndTypePair>& _data_type_with_names;
172
    const std::unordered_map<std::string, IndexIterator*>& _iterators;
173
    std::shared_ptr<IndexQueryContext> _context;
174
    std::vector<TSearchFieldBinding> _field_bindings;
175
    std::unordered_map<std::string, const TSearchFieldBinding*> _field_binding_map;
176
    std::unordered_set<std::string> _variant_subcolumn_fields;
177
    std::unordered_map<std::string, FieldReaderBinding> _cache;
178
    std::vector<std::shared_ptr<lucene::index::IndexReader>> _readers;
179
    std::unordered_map<std::string, std::shared_ptr<lucene::index::IndexReader>> _binding_readers;
180
    std::unordered_map<std::wstring, std::shared_ptr<lucene::index::IndexReader>> _field_readers;
181
    std::vector<segment_v2::InvertedIndexCacheHandle> _searcher_cache_handles;
182
    SearchLeafQueryMapper _leaf_query_mapper;
183
};
184
185
class VariantSearchNullBitmapAdapter final : public inverted_index::query_v2::NullBitmapResolver {
186
public:
187
    explicit VariantSearchNullBitmapAdapter(const FieldReaderResolver& resolver)
188
2
            : _resolver(resolver) {}
189
190
    segment_v2::IndexIterator* iterator_for(const inverted_index::query_v2::Scorer& scorer,
191
                                            const std::string& logical_field) const override;
192
193
private:
194
    const FieldReaderResolver& _resolver;
195
};
196
197
void populate_variant_search_binding_context(
198
        const FieldReaderResolver& resolver,
199
        inverted_index::query_v2::QueryExecutionContext* exec_ctx);
200
201
inverted_index::query_v2::QueryExecutionContext build_variant_search_query_execution_context(
202
        uint32_t segment_num_rows, const FieldReaderResolver& resolver,
203
        inverted_index::query_v2::NullBitmapResolver* null_resolver);
204
205
struct VariantNestedDocMapperContext {
206
    std::string root_field;
207
    std::vector<const segment_v2::NestedGroupReader*> active_group_chain;
208
    const segment_v2::VariantColumnReader* variant_reader = nullptr;
209
    const segment_v2::NestedGroupReadProvider* read_provider = nullptr;
210
    segment_v2::ColumnIteratorOptions column_iter_opts;
211
};
212
213
Status map_variant_nested_leaf_query_to_active_group(const VariantNestedDocMapperContext& context,
214
                                                     const std::string& logical_field_name,
215
                                                     inverted_index::query_v2::QueryPtr* query);
216
217
inverted_index::query_v2::QueryPtr make_variant_nested_doc_mapping_query(
218
        inverted_index::query_v2::QueryPtr child_query,
219
        std::vector<const segment_v2::NestedGroupReader*> child_to_parent_chain,
220
        const segment_v2::NestedGroupReadProvider* read_provider,
221
        segment_v2::ColumnIteratorOptions column_iter_opts);
222
223
class VariantNestedSearchEvaluator {
224
public:
225
    explicit VariantNestedSearchEvaluator(const FunctionSearch& function_search)
226
7
            : _function_search(function_search) {}
227
228
    Status evaluate(const TSearchParam& search_param, const TSearchClause& nested_clause,
229
                    const std::shared_ptr<segment_v2::IndexQueryContext>& context,
230
                    FieldReaderResolver& resolver, uint32_t num_rows,
231
                    const IndexExecContext* index_exec_ctx,
232
                    const std::unordered_map<std::string, int>& field_name_to_column_id,
233
                    std::shared_ptr<roaring::Roaring>& result_bitmap) const;
234
235
private:
236
    const FunctionSearch& _function_search;
237
};
238
239
} // namespace doris