be/src/format_v2/parquet/parquet_profile.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "format_v2/parquet/parquet_profile.h" |
19 | | |
20 | | #include "format_v2/parquet/parquet_statistics.h" |
21 | | |
22 | | namespace doris::format::parquet { |
23 | | |
24 | 109 | void ParquetProfile::init(RuntimeProfile* profile) { |
25 | 109 | if (profile == nullptr) { |
26 | 82 | return; |
27 | 82 | } |
28 | | |
29 | 27 | static const char* parquet_profile = "ParquetReader"; |
30 | 27 | ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); |
31 | | |
32 | 27 | filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, |
33 | 27 | parquet_profile, 1); |
34 | 27 | filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( |
35 | 27 | profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, parquet_profile, 1); |
36 | 27 | filtered_row_groups_by_dictionary = ADD_CHILD_COUNTER_WITH_LEVEL( |
37 | 27 | profile, "RowGroupsFilteredByDictionary", TUnit::UNIT, parquet_profile, 1); |
38 | 27 | filtered_row_groups_by_bloom_filter = ADD_CHILD_COUNTER_WITH_LEVEL( |
39 | 27 | profile, "RowGroupsFilteredByBloomFilter", TUnit::UNIT, parquet_profile, 1); |
40 | 27 | filtered_row_groups_by_page_index = ADD_CHILD_COUNTER_WITH_LEVEL( |
41 | 27 | profile, "RowGroupsFilteredByPageIndex", TUnit::UNIT, parquet_profile, 1); |
42 | 27 | to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsReadNum", TUnit::UNIT, |
43 | 27 | parquet_profile, 1); |
44 | 27 | total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsTotalNum", TUnit::UNIT, |
45 | 27 | parquet_profile, 1); |
46 | 27 | selected_row_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRowRanges", TUnit::UNIT, |
47 | 27 | parquet_profile, 1); |
48 | 27 | filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByGroup", TUnit::UNIT, |
49 | 27 | parquet_profile, 1); |
50 | 27 | filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByPage", TUnit::UNIT, |
51 | 27 | parquet_profile, 1); |
52 | 27 | pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL( |
53 | 27 | profile, "PagesSkippedByDataPageFilter", TUnit::UNIT, parquet_profile, 1); |
54 | 27 | data_page_filter_skip_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DataPageFilterSkipBytes", |
55 | 27 | TUnit::BYTES, parquet_profile, 1); |
56 | 27 | selected_rows = |
57 | 27 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); |
58 | 27 | rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", |
59 | 27 | TUnit::UNIT, parquet_profile, 1); |
60 | 27 | total_batches = |
61 | 27 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); |
62 | 27 | empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", |
63 | 27 | TUnit::UNIT, parquet_profile, 1); |
64 | 27 | range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", |
65 | 27 | TUnit::UNIT, parquet_profile, 1); |
66 | 27 | reader_read_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderReadRows", TUnit::UNIT, |
67 | 27 | parquet_profile, 1); |
68 | 27 | reader_skip_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSkipRows", TUnit::UNIT, |
69 | 27 | parquet_profile, 1); |
70 | 27 | reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, |
71 | 27 | parquet_profile, 1); |
72 | 27 | arrow_read_records_time = |
73 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); |
74 | 27 | materialization_time = |
75 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); |
76 | 27 | lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", |
77 | 27 | TUnit::UNIT, parquet_profile, 1); |
78 | 27 | filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, |
79 | 27 | parquet_profile, 1); |
80 | 27 | raw_rows_read = |
81 | 27 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); |
82 | 27 | column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); |
83 | 27 | parse_meta_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseMetaTime", parquet_profile, 1); |
84 | 27 | parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); |
85 | 27 | file_reader_create_time = |
86 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); |
87 | 27 | open_file_num = |
88 | 27 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); |
89 | 27 | page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); |
90 | 27 | page_index_filter_time = |
91 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); |
92 | 27 | read_page_index_time = |
93 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexReadTime", parquet_profile, 1); |
94 | 27 | parse_page_index_time = |
95 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexParseTime", parquet_profile, 1); |
96 | 27 | expr_zonemap_unusable = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExprZoneMapUnusableEvals", |
97 | 27 | TUnit::UNIT, parquet_profile, 1); |
98 | 27 | in_zonemap_point_check = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapPointCheckCount", |
99 | 27 | TUnit::UNIT, parquet_profile, 1); |
100 | 27 | in_zonemap_range_only = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapRangeOnlyCount", |
101 | 27 | TUnit::UNIT, parquet_profile, 1); |
102 | 27 | row_group_filter_time = |
103 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); |
104 | 27 | file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); |
105 | 27 | file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); |
106 | 27 | decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); |
107 | 27 | decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, |
108 | 27 | parquet_profile, 1); |
109 | 27 | page_read_counter = |
110 | 27 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageReadCount", TUnit::UNIT, parquet_profile, 1); |
111 | 27 | page_cache_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheWriteCount", |
112 | 27 | TUnit::UNIT, parquet_profile, 1); |
113 | 27 | page_cache_compressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
114 | 27 | profile, "PageCacheCompressedWriteCount", TUnit::UNIT, parquet_profile, 1); |
115 | 27 | page_cache_decompressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
116 | 27 | profile, "PageCacheDecompressedWriteCount", TUnit::UNIT, parquet_profile, 1); |
117 | 27 | page_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheHitCount", TUnit::UNIT, |
118 | 27 | parquet_profile, 1); |
119 | 27 | page_cache_missing_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheMissingCount", |
120 | 27 | TUnit::UNIT, parquet_profile, 1); |
121 | 27 | page_cache_compressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
122 | 27 | profile, "PageCacheCompressedHitCount", TUnit::UNIT, parquet_profile, 1); |
123 | 27 | page_cache_decompressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
124 | 27 | profile, "PageCacheDecompressedHitCount", TUnit::UNIT, parquet_profile, 1); |
125 | 27 | decode_header_time = |
126 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderDecodeTime", parquet_profile, 1); |
127 | 27 | read_page_header_time = |
128 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderReadTime", parquet_profile, 1); |
129 | 27 | decode_value_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeValueTime", parquet_profile, 1); |
130 | 27 | decode_dict_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeDictTime", parquet_profile, 1); |
131 | 27 | decode_level_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeLevelTime", parquet_profile, 1); |
132 | 27 | decode_null_map_time = |
133 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeNullMapTime", parquet_profile, 1); |
134 | 27 | skip_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SkipPageHeaderNum", TUnit::UNIT, |
135 | 27 | parquet_profile, 1); |
136 | 27 | parse_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ParsePageHeaderNum", TUnit::UNIT, |
137 | 27 | parquet_profile, 1); |
138 | 27 | predicate_filter_time = |
139 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); |
140 | 27 | dict_filter_rewrite_time = |
141 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); |
142 | 27 | convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); |
143 | 27 | bloom_filter_read_time = |
144 | 27 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); |
145 | 27 | } |
146 | | |
147 | 27 | void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_stats) const { |
148 | 27 | COUNTER_UPDATE(filtered_row_groups, |
149 | 27 | pruning_stats.total_row_groups - pruning_stats.selected_row_groups); |
150 | 27 | COUNTER_UPDATE(filtered_row_groups_by_min_max, pruning_stats.filtered_row_groups_by_statistics); |
151 | 27 | COUNTER_UPDATE(filtered_row_groups_by_dictionary, |
152 | 27 | pruning_stats.filtered_row_groups_by_dictionary); |
153 | 27 | COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, |
154 | 27 | pruning_stats.filtered_row_groups_by_bloom_filter); |
155 | 27 | COUNTER_UPDATE(filtered_row_groups_by_page_index, |
156 | 27 | pruning_stats.filtered_row_groups_by_page_index); |
157 | 27 | COUNTER_UPDATE(to_read_row_groups, pruning_stats.selected_row_groups); |
158 | 27 | COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); |
159 | 27 | COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); |
160 | 27 | COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); |
161 | 27 | COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); |
162 | 27 | COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); |
163 | 27 | COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); |
164 | 27 | COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); |
165 | 27 | COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); |
166 | 27 | COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); |
167 | 27 | COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); |
168 | 27 | COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); |
169 | 27 | COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); |
170 | 27 | } |
171 | | |
172 | 110 | ParquetPageSkipProfile ParquetProfile::page_skip_profile() const { |
173 | 110 | return { |
174 | 110 | .skipped_pages = pages_skipped_by_data_page_filter, |
175 | 110 | .skipped_bytes = data_page_filter_skip_bytes, |
176 | 110 | }; |
177 | 110 | } |
178 | | |
179 | 110 | ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { |
180 | 110 | return { |
181 | 110 | .reader_read_rows = reader_read_rows, |
182 | 110 | .reader_skip_rows = reader_skip_rows, |
183 | 110 | .reader_select_rows = reader_select_rows, |
184 | 110 | .arrow_read_records_time = arrow_read_records_time, |
185 | 110 | .materialization_time = materialization_time, |
186 | 110 | }; |
187 | 110 | } |
188 | | |
189 | 110 | ParquetScanProfile ParquetProfile::scan_profile() const { |
190 | 110 | return { |
191 | 110 | .raw_rows_read = raw_rows_read, |
192 | 110 | .selected_rows = selected_rows, |
193 | 110 | .rows_filtered_by_conjunct = rows_filtered_by_conjunct, |
194 | 110 | .lazy_read_filtered_rows = lazy_read_filtered_rows, |
195 | 110 | .total_batches = total_batches, |
196 | 110 | .empty_selection_batches = empty_selection_batches, |
197 | 110 | .range_gap_skipped_rows = range_gap_skipped_rows, |
198 | 110 | .column_read_time = column_read_time, |
199 | 110 | .predicate_filter_time = predicate_filter_time, |
200 | 110 | .column_reader_profile = column_reader_profile(), |
201 | 110 | }; |
202 | 110 | } |
203 | | |
204 | | } // namespace doris::format::parquet |