be/src/format_v2/parquet/parquet_reader.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // http://www.apache.org/licenses/LICENSE-2.0 |
9 | | // Unless required by applicable law or agreed to in writing, |
10 | | // software distributed under the License is distributed on an |
11 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
12 | | // KIND, either express or implied. See the License for the |
13 | | // specific language governing permissions and limitations |
14 | | // under the License. |
15 | | |
16 | | #pragma once |
17 | | |
18 | | #include <memory> |
19 | | #include <optional> |
20 | | #include <string> |
21 | | #include <vector> |
22 | | |
23 | | #include "common/status.h" |
24 | | #include "format_v2/file_reader.h" |
25 | | #include "format_v2/parquet/parquet_column_schema.h" |
26 | | #include "format_v2/parquet/parquet_file_context.h" |
27 | | #include "format_v2/parquet/parquet_profile.h" |
28 | | #include "format_v2/parquet/parquet_scan.h" |
29 | | |
30 | | namespace doris { |
31 | | namespace io { |
32 | | struct IOContext; |
33 | | } // namespace io |
34 | | } // namespace doris |
35 | | |
36 | | namespace doris::format::parquet { |
37 | | |
38 | | struct ParquetReaderScanState; |
39 | | |
40 | | // ============================================================================ |
41 | | // ============================================================================ |
42 | | // init() -> get_schema() -> open(request) -> get_block() [loop] -> close() |
43 | | // ============================================================================ |
44 | | class ParquetReader : public format::FileReader { |
45 | | public: |
46 | | ParquetReader(std::shared_ptr<io::FileSystemProperties>& system_properties, |
47 | | std::unique_ptr<io::FileDescription>& file_description, |
48 | | std::shared_ptr<io::IOContext> io_ctx, RuntimeProfile* profile, |
49 | | std::optional<format::GlobalRowIdContext> global_rowid_context = std::nullopt, |
50 | | bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false, |
51 | | std::string hive_parquet_time_zone = ""); |
52 | | ~ParquetReader() override; |
53 | | |
54 | | Status init(RuntimeState* state) override; |
55 | | |
56 | | void set_batch_size(size_t batch_size) override; |
57 | | |
58 | | Status get_schema(std::vector<format::ColumnDefinition>* file_schema) const override; |
59 | | |
60 | | std::unique_ptr<format::TableColumnMapper> create_column_mapper( |
61 | | format::TableColumnMapperOptions options) const override; |
62 | | |
63 | | Status open(std::shared_ptr<format::FileScanRequest> request) override; |
64 | | |
65 | 1 | bool supports_scan_request_refresh() const override { return true; } |
66 | | |
67 | | Status queue_scan_request(std::shared_ptr<format::FileScanRequest> request) override; |
68 | | |
69 | | Status get_block(Block* file_block, size_t* rows, bool* eof) override; |
70 | | |
71 | | Status get_aggregate_result(const format::FileAggregateRequest& request, |
72 | | format::FileAggregateResult* result) override; |
73 | | |
74 | | void set_condition_cache_context(std::shared_ptr<ConditionCacheContext> ctx) override; |
75 | | |
76 | | int64_t get_total_rows() const override; |
77 | | |
78 | | Status close() override; |
79 | | |
80 | | protected: |
81 | | void _init_profile() override; |
82 | | |
83 | | private: |
84 | | void _sync_page_cache_profile(); |
85 | | bool _should_stop() const; |
86 | | Status _stop_status_if_requested(const Status& status) const; |
87 | | |
88 | | void _fill_column_definition(const ParquetColumnSchema& column_schema, |
89 | | format::ColumnDefinition* field) const; |
90 | | |
91 | | std::unique_ptr<ParquetReaderScanState> |
92 | | _state; // complete scan state (file_context + schema + scheduler) |
93 | | ParquetProfile _parquet_profile; // RuntimeProfile counter set |
94 | | ParquetPageCacheStats _reported_page_cache_stats; |
95 | | std::optional<format::GlobalRowIdContext> _global_rowid_context; // global RowId context |
96 | | size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; |
97 | | bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ |
98 | | bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY |
99 | | std::string _hive_parquet_time_zone; // explicit INT96 timezone; empty disables conversion |
100 | | }; |
101 | | |
102 | | } // namespace doris::format::parquet |