Coverage Report

Created: 2026-02-27 02:43

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/olap/tablet_reader.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "olap/tablet_reader.h"
19
20
#include <gen_cpp/olap_file.pb.h>
21
#include <gen_cpp/segment_v2.pb.h>
22
#include <thrift/protocol/TDebugProtocol.h>
23
24
#include <algorithm>
25
#include <functional>
26
#include <iterator>
27
#include <memory>
28
#include <numeric>
29
#include <ostream>
30
#include <shared_mutex>
31
32
#include "common/compiler_util.h" // IWYU pragma: keep
33
#include "common/config.h"
34
#include "common/exception.h"
35
#include "common/logging.h"
36
#include "common/status.h"
37
#include "exprs/bitmapfilter_predicate.h"
38
#include "exprs/bloom_filter_func.h"
39
#include "exprs/create_predicate_function.h"
40
#include "exprs/hybrid_set.h"
41
#include "olap/column_predicate.h"
42
#include "olap/itoken_extractor.h"
43
#include "olap/like_column_predicate.h"
44
#include "olap/olap_common.h"
45
#include "olap/olap_define.h"
46
#include "olap/predicate_creator.h"
47
#include "olap/row_cursor.h"
48
#include "olap/rowset/segment_v2/bloom_filter.h"
49
#include "olap/schema.h"
50
#include "olap/tablet.h"
51
#include "olap/tablet_meta.h"
52
#include "olap/tablet_schema.h"
53
#include "runtime/query_context.h"
54
#include "runtime/runtime_predicate.h"
55
#include "runtime/runtime_state.h"
56
#include "vec/common/arena.h"
57
#include "vec/common/variant_util.h"
58
#include "vec/core/block.h"
59
60
namespace doris {
61
#include "common/compile_check_begin.h"
62
using namespace ErrorCode;
63
64
361
void TabletReader::ReaderParams::check_validation() const {
65
361
    if (UNLIKELY(version.first == -1 && is_segcompaction == false)) {
66
0
        throw Exception(Status::FatalError("version is not set. tablet={}", tablet->tablet_id()));
67
0
    }
68
361
}
69
70
0
std::string TabletReader::ReaderParams::to_string() const {
71
0
    std::stringstream ss;
72
0
    ss << "tablet=" << tablet->tablet_id() << " reader_type=" << int(reader_type)
73
0
       << " aggregation=" << aggregation << " version=" << version
74
0
       << " start_key_include=" << start_key_include << " end_key_include=" << end_key_include;
75
76
0
    for (const auto& key : start_key) {
77
0
        ss << " keys=" << key;
78
0
    }
79
80
0
    for (const auto& key : end_key) {
81
0
        ss << " end_keys=" << key;
82
0
    }
83
84
0
    return ss.str();
85
0
}
86
87
361
Status TabletReader::init(const ReaderParams& read_params) {
88
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns);
89
90
361
    Status res = _init_params(read_params);
91
361
    if (!res.ok()) {
92
0
        LOG(WARNING) << "fail to init reader when init params. res:" << res
93
0
                     << ", tablet_id:" << read_params.tablet->tablet_id()
94
0
                     << ", schema_hash:" << read_params.tablet->schema_hash()
95
0
                     << ", reader type:" << int(read_params.reader_type)
96
0
                     << ", version:" << read_params.version;
97
0
    }
98
361
    return res;
99
361
}
100
101
339
Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
102
339
    SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns);
103
339
    if (read_params.rs_splits.empty()) {
104
0
        return Status::InternalError("fail to acquire data sources. tablet={}",
105
0
                                     _tablet->tablet_id());
106
0
    }
107
108
339
    bool eof = false;
109
339
    bool is_lower_key_included = _keys_param.start_key_include;
110
339
    bool is_upper_key_included = _keys_param.end_key_include;
111
112
339
    for (int i = 0; i < _keys_param.start_keys.size(); ++i) {
113
        // lower bound
114
0
        RowCursor& start_key = _keys_param.start_keys[i];
115
0
        RowCursor& end_key = _keys_param.end_keys[i];
116
117
0
        if (!is_lower_key_included) {
118
0
            if (compare_row_key(start_key, end_key) >= 0) {
119
0
                VLOG_NOTICE << "return EOF when lower key not include"
120
0
                            << ", start_key=" << start_key.to_string()
121
0
                            << ", end_key=" << end_key.to_string();
122
0
                eof = true;
123
0
                break;
124
0
            }
125
0
        } else {
126
0
            if (compare_row_key(start_key, end_key) > 0) {
127
0
                VLOG_NOTICE << "return EOF when lower key include="
128
0
                            << ", start_key=" << start_key.to_string()
129
0
                            << ", end_key=" << end_key.to_string();
130
0
                eof = true;
131
0
                break;
132
0
            }
133
0
        }
134
135
0
        _is_lower_keys_included.push_back(is_lower_key_included);
136
0
        _is_upper_keys_included.push_back(is_upper_key_included);
137
0
    }
138
139
339
    if (eof) {
140
0
        return Status::EndOfFile("reach end of scan range. tablet={}", _tablet->tablet_id());
141
0
    }
142
143
339
    bool need_ordered_result = true;
144
339
    if (read_params.reader_type == ReaderType::READER_QUERY) {
145
0
        if (_tablet_schema->keys_type() == DUP_KEYS) {
146
            // duplicated keys are allowed, no need to merge sort keys in rowset
147
0
            need_ordered_result = false;
148
0
        }
149
0
        if (_tablet_schema->keys_type() == UNIQUE_KEYS &&
150
0
            _tablet->enable_unique_key_merge_on_write()) {
151
            // unique keys with merge on write, no need to merge sort keys in rowset
152
0
            need_ordered_result = false;
153
0
        }
154
0
        if (_aggregation) {
155
            // compute engine will aggregate rows with the same key,
156
            // it's ok for rowset to return unordered result
157
0
            need_ordered_result = false;
158
0
        }
159
160
0
        if (_direct_mode) {
161
            // direct mode indicates that the storage layer does not need to merge,
162
            // it's ok for rowset to return unordered result
163
0
            need_ordered_result = false;
164
0
        }
165
166
0
        if (read_params.read_orderby_key) {
167
0
            need_ordered_result = true;
168
0
        }
169
0
    }
170
171
339
    _reader_context.reader_type = read_params.reader_type;
172
339
    _reader_context.version = read_params.version;
173
339
    _reader_context.tablet_schema = _tablet_schema;
174
339
    _reader_context.need_ordered_result = need_ordered_result;
175
339
    _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids;
176
339
    _reader_context.topn_filter_target_node_id = read_params.topn_filter_target_node_id;
177
339
    _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse;
178
339
    _reader_context.read_orderby_key_limit = read_params.read_orderby_key_limit;
179
339
    _reader_context.filter_block_conjuncts = read_params.filter_block_conjuncts;
180
339
    _reader_context.return_columns = &_return_columns;
181
339
    _reader_context.read_orderby_key_columns =
182
339
            !_orderby_key_columns.empty() ? &_orderby_key_columns : nullptr;
183
339
    _reader_context.predicates = &_col_predicates;
184
339
    _reader_context.value_predicates = &_value_col_predicates;
185
339
    _reader_context.lower_bound_keys = &_keys_param.start_keys;
186
339
    _reader_context.is_lower_keys_included = &_is_lower_keys_included;
187
339
    _reader_context.upper_bound_keys = &_keys_param.end_keys;
188
339
    _reader_context.is_upper_keys_included = &_is_upper_keys_included;
189
339
    _reader_context.delete_handler = &_delete_handler;
190
339
    _reader_context.stats = &_stats;
191
339
    _reader_context.use_page_cache = read_params.use_page_cache;
192
339
    _reader_context.sequence_id_idx = _sequence_col_idx;
193
339
    _reader_context.is_unique = tablet()->keys_type() == UNIQUE_KEYS;
194
339
    _reader_context.merged_rows = &_merged_rows;
195
339
    _reader_context.delete_bitmap = read_params.delete_bitmap;
196
339
    _reader_context.enable_unique_key_merge_on_write = tablet()->enable_unique_key_merge_on_write();
197
339
    _reader_context.record_rowids = read_params.record_rowids;
198
339
    _reader_context.rowid_conversion = read_params.rowid_conversion;
199
339
    _reader_context.is_key_column_group = read_params.is_key_column_group;
200
339
    _reader_context.remaining_conjunct_roots = read_params.remaining_conjunct_roots;
201
339
    _reader_context.common_expr_ctxs_push_down = read_params.common_expr_ctxs_push_down;
202
339
    _reader_context.output_columns = &read_params.output_columns;
203
339
    _reader_context.push_down_agg_type_opt = read_params.push_down_agg_type_opt;
204
339
    _reader_context.ttl_seconds = _tablet->ttl_seconds();
205
339
    _reader_context.score_runtime = read_params.score_runtime;
206
339
    _reader_context.collection_statistics = read_params.collection_statistics;
207
208
339
    _reader_context.virtual_column_exprs = read_params.virtual_column_exprs;
209
339
    _reader_context.vir_cid_to_idx_in_block = read_params.vir_cid_to_idx_in_block;
210
339
    _reader_context.vir_col_idx_to_type = read_params.vir_col_idx_to_type;
211
339
    _reader_context.ann_topn_runtime = read_params.ann_topn_runtime;
212
213
339
    _reader_context.condition_cache_digest = read_params.condition_cache_digest;
214
339
    _reader_context.all_access_paths = read_params.all_access_paths;
215
339
    _reader_context.predicate_access_paths = read_params.predicate_access_paths;
216
217
339
    return Status::OK();
218
339
}
219
220
0
TabletColumn TabletReader::materialize_column(const TabletColumn& orig) {
221
0
    if (!orig.is_variant_type()) {
222
0
        return orig;
223
0
    }
224
0
    TabletColumn column_with_cast_type = orig;
225
0
    auto cast_type = _reader_context.target_cast_type_for_variants.at(orig.name());
226
0
    return vectorized::variant_util::get_column_by_type(
227
0
            cast_type, orig.name(),
228
0
            {
229
0
                    .unique_id = orig.unique_id(),
230
0
                    .parent_unique_id = orig.parent_unique_id(),
231
0
                    .path_info = *orig.path_info_ptr(),
232
0
            });
233
0
}
234
235
361
Status TabletReader::_init_params(const ReaderParams& read_params) {
236
361
    read_params.check_validation();
237
238
361
    _direct_mode = read_params.direct_mode;
239
361
    _aggregation = read_params.aggregation;
240
361
    _reader_type = read_params.reader_type;
241
361
    _tablet = read_params.tablet;
242
361
    _tablet_schema = read_params.tablet_schema;
243
361
    _reader_context.runtime_state = read_params.runtime_state;
244
361
    _reader_context.target_cast_type_for_variants = read_params.target_cast_type_for_variants;
245
246
361
    RETURN_IF_ERROR(_init_conditions_param(read_params));
247
248
361
    Status res = _init_delete_condition(read_params);
249
361
    if (!res.ok()) {
250
0
        LOG(WARNING) << "fail to init delete param. res = " << res;
251
0
        return res;
252
0
    }
253
254
361
    res = _init_return_columns(read_params);
255
361
    if (!res.ok()) {
256
0
        LOG(WARNING) << "fail to init return columns. res = " << res;
257
0
        return res;
258
0
    }
259
260
361
    res = _init_keys_param(read_params);
261
361
    if (!res.ok()) {
262
0
        LOG(WARNING) << "fail to init keys param. res=" << res;
263
0
        return res;
264
0
    }
265
361
    res = _init_orderby_keys_param(read_params);
266
361
    if (!res.ok()) {
267
0
        LOG(WARNING) << "fail to init orderby keys param. res=" << res;
268
0
        return res;
269
0
    }
270
361
    if (_tablet_schema->has_sequence_col()) {
271
8
        auto sequence_col_idx = _tablet_schema->sequence_col_idx();
272
8
        DCHECK_NE(sequence_col_idx, -1);
273
16
        for (auto col : _return_columns) {
274
            // query has sequence col
275
16
            if (col == sequence_col_idx) {
276
4
                _sequence_col_idx = sequence_col_idx;
277
4
                break;
278
4
            }
279
16
        }
280
8
    }
281
282
361
    return res;
283
361
}
284
285
361
Status TabletReader::_init_return_columns(const ReaderParams& read_params) {
286
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns);
287
361
    if (read_params.reader_type == ReaderType::READER_QUERY) {
288
22
        _return_columns = read_params.return_columns;
289
22
        _tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set;
290
37
        for (auto id : read_params.return_columns) {
291
37
            if (_tablet_schema->column(id).is_key()) {
292
22
                _key_cids.push_back(id);
293
22
            } else {
294
15
                _value_cids.push_back(id);
295
15
            }
296
37
        }
297
339
    } else if (read_params.return_columns.empty()) {
298
0
        for (uint32_t i = 0; i < _tablet_schema->num_columns(); ++i) {
299
0
            _return_columns.push_back(i);
300
0
            if (_tablet_schema->column(i).is_key()) {
301
0
                _key_cids.push_back(i);
302
0
            } else {
303
0
                _value_cids.push_back(i);
304
0
            }
305
0
        }
306
0
        VLOG_NOTICE << "return column is empty, using full column as default.";
307
339
    } else if ((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION ||
308
339
                read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION ||
309
339
                read_params.reader_type == ReaderType::READER_BASE_COMPACTION ||
310
339
                read_params.reader_type == ReaderType::READER_FULL_COMPACTION ||
311
339
                read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION ||
312
339
                read_params.reader_type == ReaderType::READER_ALTER_TABLE) &&
313
339
               !read_params.return_columns.empty()) {
314
339
        _return_columns = read_params.return_columns;
315
1.07k
        for (auto id : read_params.return_columns) {
316
1.07k
            if (_tablet_schema->column(id).is_key()) {
317
115
                _key_cids.push_back(id);
318
961
            } else {
319
961
                _value_cids.push_back(id);
320
961
            }
321
1.07k
        }
322
339
    } else if (read_params.reader_type == ReaderType::READER_CHECKSUM) {
323
0
        _return_columns = read_params.return_columns;
324
0
        for (auto id : read_params.return_columns) {
325
0
            if (_tablet_schema->column(id).is_key()) {
326
0
                _key_cids.push_back(id);
327
0
            } else {
328
0
                _value_cids.push_back(id);
329
0
            }
330
0
        }
331
0
    } else {
332
0
        return Status::Error<INVALID_ARGUMENT>(
333
0
                "fail to init return columns. reader_type={}, return_columns_size={}",
334
0
                int(read_params.reader_type), read_params.return_columns.size());
335
0
    }
336
337
361
    std::sort(_key_cids.begin(), _key_cids.end(), std::greater<>());
338
339
361
    return Status::OK();
340
361
}
341
342
361
Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
343
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns);
344
361
    if (read_params.start_key.empty()) {
345
361
        return Status::OK();
346
361
    }
347
348
0
    _keys_param.start_key_include = read_params.start_key_include;
349
0
    _keys_param.end_key_include = read_params.end_key_include;
350
351
0
    size_t start_key_size = read_params.start_key.size();
352
    //_keys_param.start_keys.resize(start_key_size);
353
0
    std::vector<RowCursor>(start_key_size).swap(_keys_param.start_keys);
354
355
0
    size_t scan_key_size = read_params.start_key.front().size();
356
0
    if (scan_key_size > _tablet_schema->num_columns()) {
357
0
        return Status::Error<INVALID_ARGUMENT>(
358
0
                "Input param are invalid. Column count is bigger than num_columns of schema. "
359
0
                "column_count={}, schema.num_columns={}",
360
0
                scan_key_size, _tablet_schema->num_columns());
361
0
    }
362
363
0
    std::vector<uint32_t> columns(scan_key_size);
364
0
    std::iota(columns.begin(), columns.end(), 0);
365
366
0
    std::shared_ptr<Schema> schema = std::make_shared<Schema>(_tablet_schema->columns(), columns);
367
368
0
    for (size_t i = 0; i < start_key_size; ++i) {
369
0
        if (read_params.start_key[i].size() != scan_key_size) {
370
0
            return Status::Error<INVALID_ARGUMENT>(
371
0
                    "The start_key.at({}).size={}, not equals the scan_key_size={}", i,
372
0
                    read_params.start_key[i].size(), scan_key_size);
373
0
        }
374
375
0
        Status res = _keys_param.start_keys[i].init_scan_key(
376
0
                _tablet_schema, read_params.start_key[i].values(), schema);
377
0
        if (!res.ok()) {
378
0
            LOG(WARNING) << "fail to init row cursor. res = " << res;
379
0
            return res;
380
0
        }
381
0
        res = _keys_param.start_keys[i].from_tuple(read_params.start_key[i]);
382
0
        if (!res.ok()) {
383
0
            LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << "key_index=" << i;
384
0
            return res;
385
0
        }
386
0
    }
387
388
0
    size_t end_key_size = read_params.end_key.size();
389
    //_keys_param.end_keys.resize(end_key_size);
390
0
    std::vector<RowCursor>(end_key_size).swap(_keys_param.end_keys);
391
0
    for (size_t i = 0; i < end_key_size; ++i) {
392
0
        if (read_params.end_key[i].size() != scan_key_size) {
393
0
            return Status::Error<INVALID_ARGUMENT>(
394
0
                    "The end_key.at({}).size={}, not equals the scan_key_size={}", i,
395
0
                    read_params.end_key[i].size(), scan_key_size);
396
0
        }
397
398
0
        Status res = _keys_param.end_keys[i].init_scan_key(_tablet_schema,
399
0
                                                           read_params.end_key[i].values(), schema);
400
0
        if (!res.ok()) {
401
0
            LOG(WARNING) << "fail to init row cursor. res = " << res;
402
0
            return res;
403
0
        }
404
405
0
        res = _keys_param.end_keys[i].from_tuple(read_params.end_key[i]);
406
0
        if (!res.ok()) {
407
0
            LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << " key_index=" << i;
408
0
            return res;
409
0
        }
410
0
    }
411
412
    //TODO:check the valid of start_key and end_key.(eg. start_key <= end_key)
413
414
0
    return Status::OK();
415
0
}
416
417
361
Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
418
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns);
419
    // UNIQUE_KEYS will compare all keys as before
420
361
    if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS &&
421
299
                                                    _tablet->enable_unique_key_merge_on_write())) {
422
299
        if (!_tablet_schema->cluster_key_uids().empty()) {
423
0
            if (read_params.read_orderby_key_num_prefix_columns >
424
0
                _tablet_schema->cluster_key_uids().size()) {
425
0
                return Status::Error<ErrorCode::INTERNAL_ERROR>(
426
0
                        "read_orderby_key_num_prefix_columns={} > cluster_keys.size()={}",
427
0
                        read_params.read_orderby_key_num_prefix_columns,
428
0
                        _tablet_schema->cluster_key_uids().size());
429
0
            }
430
0
            for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
431
0
                auto cid = _tablet_schema->cluster_key_uids()[i];
432
0
                auto index = _tablet_schema->field_index(cid);
433
0
                if (index < 0) {
434
0
                    return Status::Error<ErrorCode::INTERNAL_ERROR>(
435
0
                            "could not find cluster key column with unique_id=" +
436
0
                            std::to_string(cid) +
437
0
                            " in tablet schema, tablet_id=" + std::to_string(_tablet->tablet_id()));
438
0
                }
439
0
                for (uint32_t idx = 0; idx < _return_columns.size(); idx++) {
440
0
                    if (_return_columns[idx] == index) {
441
0
                        _orderby_key_columns.push_back(idx);
442
0
                        break;
443
0
                    }
444
0
                }
445
0
            }
446
299
        } else {
447
            // find index in vector _return_columns
448
            //   for the read_orderby_key_num_prefix_columns orderby keys
449
299
            for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
450
0
                for (uint32_t idx = 0; idx < _return_columns.size(); idx++) {
451
0
                    if (_return_columns[idx] == i) {
452
0
                        _orderby_key_columns.push_back(idx);
453
0
                        break;
454
0
                    }
455
0
                }
456
0
            }
457
299
        }
458
299
        if (read_params.read_orderby_key_num_prefix_columns != _orderby_key_columns.size()) {
459
0
            return Status::Error<ErrorCode::INTERNAL_ERROR>(
460
0
                    "read_orderby_key_num_prefix_columns != _orderby_key_columns.size, "
461
0
                    "read_params.read_orderby_key_num_prefix_columns={}, "
462
0
                    "_orderby_key_columns.size()={}",
463
0
                    read_params.read_orderby_key_num_prefix_columns, _orderby_key_columns.size());
464
0
        }
465
299
    }
466
467
361
    return Status::OK();
468
361
}
469
470
361
Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
471
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns);
472
361
    std::vector<std::shared_ptr<ColumnPredicate>> predicates;
473
361
    std::copy(read_params.predicates.cbegin(), read_params.predicates.cend(),
474
361
              std::inserter(predicates, predicates.begin()));
475
    // Function filter push down to storage engine
476
361
    auto is_like_predicate = [](std::shared_ptr<ColumnPredicate> _pred) {
477
0
        return dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(_pred.get()) != nullptr ||
478
0
               dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(_pred.get()) != nullptr;
479
0
    };
480
481
361
    for (const auto& filter : read_params.function_filters) {
482
0
        predicates.emplace_back(_parse_to_predicate(filter));
483
0
        auto pred = predicates.back();
484
485
0
        const auto& col = _tablet_schema->column(pred->column_id());
486
0
        const auto* tablet_index = _tablet_schema->get_ngram_bf_index(col.unique_id());
487
0
        if (is_like_predicate(pred) && tablet_index && config::enable_query_like_bloom_filter) {
488
0
            std::unique_ptr<segment_v2::BloomFilter> ng_bf;
489
0
            std::string pattern = pred->get_search_str();
490
0
            auto gram_bf_size = tablet_index->get_gram_bf_size();
491
0
            auto gram_size = tablet_index->get_gram_size();
492
493
0
            RETURN_IF_ERROR(segment_v2::BloomFilter::create(segment_v2::NGRAM_BLOOM_FILTER, &ng_bf,
494
0
                                                            gram_bf_size));
495
0
            NgramTokenExtractor _token_extractor(gram_size);
496
497
0
            if (_token_extractor.string_like_to_bloom_filter(pattern.data(), pattern.length(),
498
0
                                                             *ng_bf)) {
499
0
                pred->set_page_ng_bf(std::move(ng_bf));
500
0
            }
501
0
        }
502
0
    }
503
504
361
    for (auto predicate : predicates) {
505
0
        auto column = _tablet_schema->column(predicate->column_id());
506
0
        if (column.aggregation() != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) {
507
0
            _value_col_predicates.push_back(predicate);
508
0
        } else {
509
0
            _col_predicates.push_back(predicate);
510
0
        }
511
0
    }
512
513
361
    return Status::OK();
514
361
}
515
516
std::shared_ptr<ColumnPredicate> TabletReader::_parse_to_predicate(
517
0
        const FunctionFilter& function_filter) {
518
0
    int32_t index = _tablet_schema->field_index(function_filter._col_name);
519
0
    if (index < 0) {
520
0
        throw Exception(Status::InternalError("Column {} not found in tablet schema",
521
0
                                              function_filter._col_name));
522
0
        return nullptr;
523
0
    }
524
0
    const TabletColumn& column = materialize_column(_tablet_schema->column(index));
525
0
    return create_column_predicate(index, std::make_shared<FunctionFilter>(function_filter),
526
0
                                   column.type(), &column);
527
0
}
528
529
361
Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
530
361
    SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns);
531
    // If it's cumu and not allow do delete when cumu
532
361
    if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION ||
533
361
        (read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
534
361
         !config::enable_delete_when_cumu_compaction)) {
535
0
        return Status::OK();
536
0
    }
537
361
    bool cumu_delete = read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
538
361
                       config::enable_delete_when_cumu_compaction;
539
    // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates.
540
    // If delete design is applied on cumu compaction, it will lose effect when doing base compaction.
541
    // `_delete_sign_available` indicates the condition where we could apply delete signs to data.
542
361
    _delete_sign_available = (((read_params.reader_type == ReaderType::READER_BASE_COMPACTION ||
543
361
                                read_params.reader_type == ReaderType::READER_FULL_COMPACTION) &&
544
361
                               config::enable_prune_delete_sign_when_base_compaction) ||
545
361
                              read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION ||
546
361
                              read_params.reader_type == ReaderType::READER_CHECKSUM);
547
548
    // `_filter_delete` indicates the condition where we should execlude deleted tuples when reading data.
549
    // However, queries will not use this condition but generate special where predicates to filter data.
550
    // (Though a lille bit confused, it is how the current logic working...)
551
361
    _filter_delete = _delete_sign_available || cumu_delete;
552
361
    return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
553
361
                                read_params.version.second);
554
361
}
555
556
Status TabletReader::init_reader_params_and_create_block(
557
        TabletSharedPtr tablet, ReaderType reader_type,
558
        const std::vector<RowsetSharedPtr>& input_rowsets,
559
0
        TabletReader::ReaderParams* reader_params, vectorized::Block* block) {
560
0
    reader_params->tablet = tablet;
561
0
    reader_params->reader_type = reader_type;
562
0
    reader_params->version =
563
0
            Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version());
564
565
0
    TabletReadSource read_source;
566
0
    for (const auto& rowset : input_rowsets) {
567
0
        RowsetReaderSharedPtr rs_reader;
568
0
        RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
569
0
        read_source.rs_splits.emplace_back(std::move(rs_reader));
570
0
    }
571
0
    read_source.fill_delete_predicates();
572
0
    reader_params->set_read_source(std::move(read_source));
573
574
0
    std::vector<RowsetMetaSharedPtr> rowset_metas(input_rowsets.size());
575
0
    std::transform(input_rowsets.begin(), input_rowsets.end(), rowset_metas.begin(),
576
0
                   [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); });
577
0
    TabletSchemaSPtr read_tablet_schema =
578
0
            tablet->tablet_schema_with_merged_max_schema_version(rowset_metas);
579
0
    TabletSchemaSPtr merge_tablet_schema = std::make_shared<TabletSchema>();
580
0
    merge_tablet_schema->copy_from(*read_tablet_schema);
581
582
    // Merge the columns in delete predicate that not in latest schema in to current tablet schema
583
0
    for (auto& del_pred : reader_params->delete_predicates) {
584
0
        merge_tablet_schema->merge_dropped_columns(*del_pred->tablet_schema());
585
0
    }
586
0
    reader_params->tablet_schema = merge_tablet_schema;
587
588
0
    reader_params->return_columns.resize(read_tablet_schema->num_columns());
589
0
    std::iota(reader_params->return_columns.begin(), reader_params->return_columns.end(), 0);
590
0
    reader_params->origin_return_columns = &reader_params->return_columns;
591
592
0
    *block = read_tablet_schema->create_block();
593
594
0
    return Status::OK();
595
0
}
596
597
#include "common/compile_check_end.h"
598
} // namespace doris