Coverage Report

Created: 2025-04-15 00:21

/root/doris/be/src/olap/tablet_reader.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/PaloInternalService_types.h>
21
#include <gen_cpp/PlanNodes_types.h>
22
#include <stddef.h>
23
#include <stdint.h>
24
25
#include <memory>
26
#include <set>
27
#include <string>
28
#include <unordered_set>
29
#include <utility>
30
#include <vector>
31
32
#include "agent/be_exec_version_manager.h"
33
#include "common/status.h"
34
#include "exprs/function_filter.h"
35
#include "gutil/strings/substitute.h"
36
#include "io/io_common.h"
37
#include "olap/delete_handler.h"
38
#include "olap/filter_olap_param.h"
39
#include "olap/iterators.h"
40
#include "olap/olap_common.h"
41
#include "olap/olap_tuple.h"
42
#include "olap/row_cursor.h"
43
#include "olap/rowid_conversion.h"
44
#include "olap/rowset/rowset.h"
45
#include "olap/rowset/rowset_meta.h"
46
#include "olap/rowset/rowset_reader.h"
47
#include "olap/rowset/rowset_reader_context.h"
48
#include "olap/tablet_fwd.h"
49
50
namespace doris {
51
52
class RuntimeState;
53
class BitmapFilterFuncBase;
54
class BloomFilterFuncBase;
55
class ColumnPredicate;
56
class DeleteBitmap;
57
class HybridSetBase;
58
class RuntimeProfile;
59
60
namespace vectorized {
61
class VCollectIterator;
62
class Block;
63
class VExpr;
64
class Arena;
65
class VExprContext;
66
} // namespace vectorized
67
68
// Used to compare row with input scan key. Scan key only contains key columns,
69
// row contains all key columns, which is superset of key columns.
70
// So we should compare the common prefix columns of lhs and rhs.
71
//
72
// NOTE: if you are not sure if you can use it, please don't use this function.
73
0
inline int compare_row_key(const RowCursor& lhs, const RowCursor& rhs) {
74
0
    auto cmp_cids = std::min(lhs.schema()->num_column_ids(), rhs.schema()->num_column_ids());
75
0
    for (uint32_t cid = 0; cid < cmp_cids; ++cid) {
76
0
        auto res = lhs.schema()->column(cid)->compare_cell(lhs.cell(cid), rhs.cell(cid));
77
0
        if (res != 0) {
78
0
            return res;
79
0
        }
80
0
    }
81
0
    return 0;
82
0
}
83
84
class TabletReader {
85
    struct KeysParam {
86
        std::string to_string() const;
87
88
        std::vector<RowCursor> start_keys;
89
        std::vector<RowCursor> end_keys;
90
        bool start_key_include = false;
91
        bool end_key_include = false;
92
    };
93
94
public:
95
    struct ReadSource {
96
        std::vector<RowSetSplits> rs_splits;
97
        std::vector<RowsetMetaSharedPtr> delete_predicates;
98
        // Fill delete predicates with `rs_splits`
99
        void fill_delete_predicates();
100
    };
101
    // Params for Reader,
102
    // mainly include tablet, data version and fetch range.
103
    struct ReaderParams {
104
0
        bool has_single_version() const {
105
0
            return (rs_splits.size() == 1 &&
106
0
                    rs_splits[0].rs_reader->rowset()->start_version() == 0 &&
107
0
                    !rs_splits[0].rs_reader->rowset()->rowset_meta()->is_segments_overlapping()) ||
108
0
                   (rs_splits.size() == 2 &&
109
0
                    rs_splits[0].rs_reader->rowset()->rowset_meta()->num_rows() == 0 &&
110
0
                    rs_splits[1].rs_reader->rowset()->start_version() == 2 &&
111
0
                    !rs_splits[1].rs_reader->rowset()->rowset_meta()->is_segments_overlapping());
112
0
        }
113
114
3
        int get_be_exec_version() const {
115
3
            if (runtime_state) {
116
0
                return runtime_state->be_exec_version();
117
0
            }
118
3
            return BeExecVersionManager::get_newest_version();
119
3
        }
120
121
342
        void set_read_source(ReadSource read_source) {
122
342
            rs_splits = std::move(read_source.rs_splits);
123
342
            delete_predicates = std::move(read_source.delete_predicates);
124
342
        }
125
126
        BaseTabletSPtr tablet;
127
        TabletSchemaSPtr tablet_schema;
128
        ReaderType reader_type = ReaderType::READER_QUERY;
129
        bool direct_mode = false;
130
        bool aggregation = false;
131
        // for compaction, schema_change, check_sum: we don't use page cache
132
        // for query and config::disable_storage_page_cache is false, we use page cache
133
        bool use_page_cache = false;
134
        Version version = Version(-1, 0);
135
136
        std::vector<OlapTuple> start_key;
137
        std::vector<OlapTuple> end_key;
138
        bool start_key_include = false;
139
        bool end_key_include = false;
140
141
        std::vector<FilterOlapParam<TCondition>> conditions;
142
        std::vector<FilterOlapParam<std::shared_ptr<BloomFilterFuncBase>>> bloom_filters;
143
        std::vector<FilterOlapParam<std::shared_ptr<BitmapFilterFuncBase>>> bitmap_filters;
144
        std::vector<FilterOlapParam<std::shared_ptr<HybridSetBase>>> in_filters;
145
        std::vector<FunctionFilter> function_filters;
146
        std::vector<RowsetMetaSharedPtr> delete_predicates;
147
        // slots that cast may be eliminated in storage layer
148
        std::map<std::string, TypeDescriptor> target_cast_type_for_variants;
149
150
        std::vector<RowSetSplits> rs_splits;
151
        // For unique key table with merge-on-write
152
        DeleteBitmap* delete_bitmap = nullptr;
153
154
        // return_columns is init from query schema
155
        std::vector<uint32_t> return_columns;
156
        // output_columns only contain columns in OrderByExprs and outputExprs
157
        std::set<int32_t> output_columns;
158
        RuntimeProfile* profile = nullptr;
159
        RuntimeState* runtime_state = nullptr;
160
161
        // use only in vec exec engine
162
        std::vector<uint32_t>* origin_return_columns = nullptr;
163
        std::unordered_set<uint32_t>* tablet_columns_convert_to_null_set = nullptr;
164
        TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE;
165
        vectorized::VExpr* remaining_vconjunct_root = nullptr;
166
        std::vector<vectorized::VExprSPtr> remaining_conjunct_roots;
167
        vectorized::VExprContextSPtrs common_expr_ctxs_push_down;
168
169
        // used for compaction to record row ids
170
        bool record_rowids = false;
171
        RowIdConversion* rowid_conversion = nullptr;
172
        std::vector<int> topn_filter_source_node_ids;
173
        int topn_filter_target_node_id = -1;
174
        // used for special optimization for query : ORDER BY key LIMIT n
175
        bool read_orderby_key = false;
176
        // used for special optimization for query : ORDER BY key DESC LIMIT n
177
        bool read_orderby_key_reverse = false;
178
        // num of columns for orderby key
179
        size_t read_orderby_key_num_prefix_columns = 0;
180
        // limit of rows for read_orderby_key
181
        size_t read_orderby_key_limit = 0;
182
        // filter_block arguments
183
        vectorized::VExprContextSPtrs filter_block_conjuncts;
184
185
        // for vertical compaction
186
        bool is_key_column_group = false;
187
        std::vector<uint32_t> key_group_cluster_key_idxes;
188
189
        bool is_segcompaction = false;
190
191
        std::vector<RowwiseIteratorUPtr>* segment_iters_ptr = nullptr;
192
193
        void check_validation() const;
194
195
        std::string to_string() const;
196
197
        int64_t batch_size = -1;
198
    };
199
200
353
    TabletReader() = default;
201
202
    virtual ~TabletReader();
203
204
    TabletReader(const TabletReader&) = delete;
205
    void operator=(const TabletReader&) = delete;
206
207
    // Initialize TabletReader with tablet, data version and fetch range.
208
    virtual Status init(const ReaderParams& read_params);
209
210
    // Read next block with aggregation.
211
    // Return OK and set `*eof` to false when next block is read
212
    // Return OK and set `*eof` to true when no more rows can be read.
213
    // Return others when unexpected error happens.
214
0
    virtual Status next_block_with_aggregation(vectorized::Block* block, bool* eof) {
215
0
        return Status::Error<ErrorCode::READER_INITIALIZE_ERROR>(
216
0
                "TabletReader not support next_block_with_aggregation");
217
0
    }
218
219
48
    virtual uint64_t merged_rows() const { return _merged_rows; }
220
221
137
    uint64_t filtered_rows() const {
222
137
        return _stats.rows_del_filtered + _stats.rows_del_by_bitmap +
223
137
               _stats.rows_conditions_filtered + _stats.rows_vec_del_cond_filtered +
224
137
               _stats.rows_vec_cond_filtered + _stats.rows_short_circuit_cond_filtered;
225
137
    }
226
227
0
    void set_batch_size(int batch_size) { _reader_context.batch_size = batch_size; }
228
229
294
    int batch_size() const { return _reader_context.batch_size; }
230
231
0
    const OlapReaderStatistics& stats() const { return _stats; }
232
0
    OlapReaderStatistics* mutable_stats() { return &_stats; }
233
234
0
    virtual void update_profile(RuntimeProfile* profile) {}
235
    static Status init_reader_params_and_create_block(
236
            TabletSharedPtr tablet, ReaderType reader_type,
237
            const std::vector<RowsetSharedPtr>& input_rowsets,
238
            TabletReader::ReaderParams* reader_params, vectorized::Block* block);
239
240
protected:
241
    friend class vectorized::VCollectIterator;
242
    friend class DeleteHandler;
243
244
    Status _init_params(const ReaderParams& read_params);
245
246
    Status _capture_rs_readers(const ReaderParams& read_params);
247
248
    bool _optimize_for_single_rowset(const std::vector<RowsetReaderSharedPtr>& rs_readers);
249
250
    Status _init_keys_param(const ReaderParams& read_params);
251
252
    Status _init_orderby_keys_param(const ReaderParams& read_params);
253
254
    Status _init_conditions_param(const ReaderParams& read_params);
255
256
    ColumnPredicate* _parse_to_predicate(
257
            const std::pair<std::string, std::shared_ptr<BloomFilterFuncBase>>& bloom_filter);
258
259
    ColumnPredicate* _parse_to_predicate(
260
            const std::pair<std::string, std::shared_ptr<BitmapFilterFuncBase>>& bitmap_filter);
261
262
    ColumnPredicate* _parse_to_predicate(
263
            const std::pair<std::string, std::shared_ptr<HybridSetBase>>& in_filter);
264
265
    virtual ColumnPredicate* _parse_to_predicate(const FunctionFilter& function_filter);
266
267
    Status _init_delete_condition(const ReaderParams& read_params);
268
269
    Status _init_return_columns(const ReaderParams& read_params);
270
271
1.20k
    const BaseTabletSPtr& tablet() { return _tablet; }
272
    // If original column is a variant type column, and it's predicate is normalized
273
    // so in order to get the real type of column predicate, we need to reset type
274
    // according to the related type in `target_cast_type_for_variants`.Since variant is not
275
    // an predicate applicable type.Otherwise return the original tablet column.
276
    // Eg. `where cast(v:a as bigint) > 1` will elimate cast, and materialize this variant column
277
    // to type bigint
278
    TabletColumn materialize_column(const TabletColumn& orig);
279
280
240
    const TabletSchema& tablet_schema() { return *_tablet_schema; }
281
282
    std::unique_ptr<vectorized::Arena> _predicate_arena;
283
    std::vector<uint32_t> _return_columns;
284
    // used for special optimization for query : ORDER BY key [ASC|DESC] LIMIT n
285
    // columns for orderby keys
286
    std::vector<uint32_t> _orderby_key_columns;
287
    // only use in outer join which change the column nullable which must keep same in
288
    // vec query engine
289
    std::unordered_set<uint32_t>* _tablet_columns_convert_to_null_set = nullptr;
290
291
    BaseTabletSPtr _tablet;
292
    RowsetReaderContext _reader_context;
293
    TabletSchemaSPtr _tablet_schema;
294
    KeysParam _keys_param;
295
    std::vector<bool> _is_lower_keys_included;
296
    std::vector<bool> _is_upper_keys_included;
297
    std::vector<ColumnPredicate*> _col_predicates;
298
    std::vector<ColumnPredicate*> _value_col_predicates;
299
    DeleteHandler _delete_handler;
300
301
    bool _aggregation = false;
302
    // for agg query, we don't need to finalize when scan agg object data
303
    ReaderType _reader_type = ReaderType::READER_QUERY;
304
    bool _next_delete_flag = false;
305
    bool _delete_sign_available = false;
306
    bool _filter_delete = false;
307
    int32_t _sequence_col_idx = -1;
308
    bool _direct_mode = false;
309
310
    std::vector<uint32_t> _key_cids;
311
    std::vector<uint32_t> _value_cids;
312
313
    uint64_t _merged_rows = 0;
314
    OlapReaderStatistics _stats;
315
};
316
317
} // namespace doris