be/src/format/generic_reader.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <gen_cpp/PlanNodes_types.h> |
21 | | |
22 | | #include "common/status.h" |
23 | | #include "exprs/vexpr_fwd.h" |
24 | | #include "runtime/descriptors.h" |
25 | | #include "storage/predicate/block_column_predicate.h" |
26 | | #include "util/profile_collector.h" |
27 | | |
28 | | namespace doris { |
29 | | class ColumnPredicate; |
30 | | } // namespace doris |
31 | | |
32 | | namespace doris { |
33 | | |
34 | | class Block; |
35 | | class VSlotRef; |
36 | | |
37 | | // Context passed from FileScanner to readers for condition cache integration. |
38 | | // On MISS: readers populate filter_result per-granule during predicate evaluation. |
39 | | // On HIT: readers skip granules where filter_result[granule] == false. |
40 | | struct ConditionCacheContext { |
41 | | bool is_hit = false; |
42 | | std::shared_ptr<std::vector<bool>> filter_result; // per-granule: true = has surviving rows |
43 | | int64_t base_granule = 0; // global granule index of the first granule in filter_result |
44 | | static constexpr int GRANULE_SIZE = 2048; |
45 | | }; |
46 | | |
47 | | // This a reader interface for all file readers. |
48 | | // A GenericReader is responsible for reading a file and return |
49 | | // a set of blocks with specified schema, |
50 | | class GenericReader : public ProfileCollector { |
51 | | public: |
52 | 3.52k | GenericReader() : _push_down_agg_type(TPushAggOp::type::NONE) {} |
53 | 2.88k | void set_push_down_agg_type(TPushAggOp::type push_down_agg_type) { |
54 | 2.88k | _push_down_agg_type = push_down_agg_type; |
55 | 2.88k | } |
56 | | |
57 | | virtual Status get_next_block(Block* block, size_t* read_rows, bool* eof) = 0; |
58 | | |
59 | | // Type is always nullable to process illegal values. |
60 | | virtual Status get_columns(std::unordered_map<std::string, DataTypePtr>* name_to_type, |
61 | 0 | std::unordered_set<std::string>* missing_cols) { |
62 | 0 | return Status::NotSupported("get_columns is not implemented"); |
63 | 0 | } |
64 | | |
65 | | // This method is responsible for initializing the resource for parsing schema. |
66 | | // It will be called before `get_parsed_schema`. |
67 | 0 | virtual Status init_schema_reader() { |
68 | 0 | return Status::NotSupported("init_schema_reader is not implemented for this reader."); |
69 | 0 | } |
70 | | // `col_types` is always nullable to process illegal values. |
71 | | virtual Status get_parsed_schema(std::vector<std::string>* col_names, |
72 | 0 | std::vector<DataTypePtr>* col_types) { |
73 | 0 | return Status::NotSupported("get_parsed_schema is not implemented for this reader."); |
74 | 0 | } |
75 | 3.52k | ~GenericReader() override = default; |
76 | | |
77 | | /// If the underlying FileReader has filled the partition&missing columns, |
78 | | /// The FileScanner does not need to fill |
79 | 5.74k | virtual bool fill_all_columns() const { return _fill_all_columns; } |
80 | | |
81 | | /// Tell the underlying FileReader the partition&missing columns, |
82 | | /// and the FileReader determine to fill columns or not. |
83 | | /// Should set _fill_all_columns = true, if fill the columns. |
84 | | virtual Status set_fill_columns( |
85 | | const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>& |
86 | | partition_columns, |
87 | 2.43k | const std::unordered_map<std::string, VExprContextSPtr>& missing_columns) { |
88 | 2.43k | return Status::OK(); |
89 | 2.43k | } |
90 | | |
91 | 567 | virtual Status close() { return Status::OK(); } |
92 | | |
93 | 54 | Status read_by_rows(const std::list<int64_t>& row_ids) { |
94 | 54 | _read_by_rows = true; |
95 | 54 | _row_ids = row_ids; |
96 | 54 | return _set_read_one_line_impl(); |
97 | 54 | } |
98 | | |
99 | | /// The reader is responsible for counting the number of rows read, |
100 | | /// because some readers, such as parquet/orc, |
101 | | /// can skip some pages/rowgroups through indexes. |
102 | 5.45k | virtual bool count_read_rows() { return false; } |
103 | | |
104 | | protected: |
105 | 0 | virtual Status _set_read_one_line_impl() { |
106 | 0 | return Status::NotSupported("read_by_rows is not implemented for this reader."); |
107 | 0 | } |
108 | | |
109 | | const size_t _MIN_BATCH_SIZE = 4064; // 4094 - 32(padding) |
110 | | |
111 | | /// Whether the underlying FileReader has filled the partition&missing columns |
112 | | bool _fill_all_columns = false; |
113 | | TPushAggOp::type _push_down_agg_type {}; |
114 | | |
115 | | public: |
116 | | // Pass condition cache context to the reader for HIT/MISS tracking. |
117 | 0 | virtual void set_condition_cache_context(std::shared_ptr<ConditionCacheContext> ctx) {} |
118 | | |
119 | | // Returns the total number of rows the reader will produce. |
120 | | // Used to pre-allocate condition cache with the correct number of granules. |
121 | 2 | virtual int64_t get_total_rows() const { return 0; } |
122 | | |
123 | | // Returns true if this reader has delete operations (e.g. Iceberg position/equality deletes, |
124 | | // Hive ACID deletes). Used to disable condition cache when deletes are present, since cached |
125 | | // granule results may become stale if delete files change between queries. |
126 | 2 | virtual bool has_delete_operations() const { return false; } |
127 | | |
128 | | protected: |
129 | | bool _read_by_rows = false; |
130 | | std::list<int64_t> _row_ids; |
131 | | |
132 | | // Cache to save some common part such as file footer. |
133 | | // Maybe null if not used |
134 | | FileMetaCache* _meta_cache = nullptr; |
135 | | }; |
136 | | |
137 | | } // namespace doris |