be/src/format/native/native_reader.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <gen_cpp/PlanNodes_types.h> |
21 | | |
22 | | #include <cstddef> |
23 | | #include <memory> |
24 | | #include <string> |
25 | | #include <unordered_map> |
26 | | #include <unordered_set> |
27 | | |
28 | | #include "common/status.h" |
29 | | #include "format/table/table_format_reader.h" |
30 | | #include "io/fs/file_reader_writer_fwd.h" |
31 | | |
32 | | namespace doris { |
33 | | class RuntimeProfile; |
34 | | class RuntimeState; |
35 | | |
36 | | namespace io { |
37 | | struct IOContext; |
38 | | } // namespace io |
39 | | } // namespace doris |
40 | | |
41 | | namespace doris { |
42 | | class Block; |
43 | | |
44 | | // Doris Native format reader. |
45 | | // it will read a sequence of Blocks encoded in Doris Native binary format. |
46 | | // |
47 | | // NOTE: current implementation is just a skeleton and will be filled step by step. |
48 | | class NativeReader : public TableFormatReader { |
49 | | public: |
50 | | ENABLE_FACTORY_CREATOR(NativeReader); |
51 | | |
52 | | NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, |
53 | | const TFileRangeDesc& range, io::IOContext* io_ctx, RuntimeState* state); |
54 | | |
55 | | NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, |
56 | | const TFileRangeDesc& range, std::shared_ptr<io::IOContext> io_ctx_holder, |
57 | | RuntimeState* state); |
58 | | |
59 | | ~NativeReader() override; |
60 | | |
61 | | // Initialize underlying file reader and any format specific state. |
62 | | Status init_reader(); |
63 | | |
64 | | Status _do_get_next_block(Block* block, size_t* read_rows, bool* eof) override; |
65 | | |
66 | | Status _get_columns_impl(std::unordered_map<std::string, DataTypePtr>* name_to_type) override; |
67 | | |
68 | | Status init_schema_reader() override; |
69 | | |
70 | | Status get_parsed_schema(std::vector<std::string>* col_names, |
71 | | std::vector<DataTypePtr>* col_types) override; |
72 | | |
73 | | Status close() override; |
74 | | |
75 | 0 | bool count_read_rows() override { return true; } |
76 | | |
77 | | protected: |
78 | 0 | void _collect_profile_before_close() override {} |
79 | 0 | Status _do_init_reader(ReaderInitContext* /*ctx*/) override { return init_reader(); } |
80 | | |
81 | | private: |
82 | | RuntimeProfile* _profile = nullptr; |
83 | | const TFileScanRangeParams& _scan_params; |
84 | | const TFileRangeDesc& _scan_range; |
85 | | |
86 | | io::FileReaderSPtr _file_reader; |
87 | | io::IOContext* _io_ctx = nullptr; |
88 | | std::shared_ptr<io::IOContext> _io_ctx_holder; |
89 | | RuntimeState* _state = nullptr; |
90 | | |
91 | | bool _eof = false; |
92 | | |
93 | | // Current read offset in the underlying file. |
94 | | int64_t _current_offset = 0; |
95 | | int64_t _file_size = 0; |
96 | | |
97 | | // Cached schema information from the first PBlock. |
98 | | bool _schema_inited = false; |
99 | | std::vector<std::string> _schema_col_names; |
100 | | std::vector<DataTypePtr> _schema_col_types; |
101 | | |
102 | | // Cached first block (serialized) to allow schema probing before data scan. |
103 | | std::string _first_block_buf; |
104 | | bool _first_block_loaded = false; |
105 | | bool _first_block_consumed = false; |
106 | | |
107 | | Status _read_next_pblock(std::string* buff, bool* eof); |
108 | | Status _init_schema_from_pblock(const PBlock& pblock); |
109 | | }; |
110 | | |
111 | | } // namespace doris |