be/src/format_v2/parquet/parquet_profile.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "format_v2/parquet/parquet_profile.h" |
19 | | |
20 | | #include "format_v2/parquet/parquet_statistics.h" |
21 | | #include "runtime/file_scan_profile.h" |
22 | | |
23 | | namespace doris::format::parquet { |
24 | | |
25 | | namespace { |
26 | | |
27 | | std::shared_ptr<RuntimeProfile::Counter> add_persistent_counter(RuntimeProfile* profile, |
28 | | const std::string& name, |
29 | | TUnit::type type, |
30 | 2.42k | const std::string& parent) { |
31 | | // A shredded Variant may be materialized after its scanner profile is destroyed. Keep the |
32 | | // counter storage alive with the column state instead of retaining a dangling profile pointer. |
33 | 2.42k | return profile->add_shared_counter(name, type, parent, 1); |
34 | 2.42k | } |
35 | | |
36 | | } // namespace |
37 | | |
38 | 644 | void ParquetProfile::init(RuntimeProfile* profile) { |
39 | 644 | if (profile == nullptr) { |
40 | 240 | return; |
41 | 240 | } |
42 | | |
43 | 404 | file_scan_profile::ensure_hierarchy(profile); |
44 | 404 | static const char* parquet_profile = "ParquetReader"; |
45 | 404 | total_time = |
46 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); |
47 | 404 | refresh_scan_request_time = |
48 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "RefreshScanRequestTime", parquet_profile, 1); |
49 | | |
50 | | // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them |
51 | | // below the format node so profile parsers and operators can attribute pruning to Parquet. |
52 | 404 | filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, |
53 | 404 | parquet_profile, 1); |
54 | 404 | filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( |
55 | 404 | profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, parquet_profile, 1); |
56 | 404 | filtered_row_groups_by_dictionary = ADD_CHILD_COUNTER_WITH_LEVEL( |
57 | 404 | profile, "RowGroupsFilteredByDictionary", TUnit::UNIT, parquet_profile, 1); |
58 | 404 | filtered_row_groups_by_bloom_filter = ADD_CHILD_COUNTER_WITH_LEVEL( |
59 | 404 | profile, "RowGroupsFilteredByBloomFilter", TUnit::UNIT, parquet_profile, 1); |
60 | 404 | filtered_row_groups_by_page_index = ADD_CHILD_COUNTER_WITH_LEVEL( |
61 | 404 | profile, "RowGroupsFilteredByPageIndex", TUnit::UNIT, parquet_profile, 1); |
62 | 404 | to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsReadNum", TUnit::UNIT, |
63 | 404 | parquet_profile, 1); |
64 | 404 | total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsTotalNum", TUnit::UNIT, |
65 | 404 | parquet_profile, 1); |
66 | 404 | selected_row_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRowRanges", TUnit::UNIT, |
67 | 404 | parquet_profile, 1); |
68 | 404 | filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByGroup", TUnit::UNIT, |
69 | 404 | parquet_profile, 1); |
70 | 404 | filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByPage", TUnit::UNIT, |
71 | 404 | parquet_profile, 1); |
72 | 404 | variant_leaf_projections = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "VariantLeafProjections", |
73 | 404 | TUnit::UNIT, parquet_profile, 1); |
74 | 404 | pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL( |
75 | 404 | profile, "PagesSkippedByDataPageFilter", TUnit::UNIT, parquet_profile, 1); |
76 | 404 | data_page_filter_skip_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DataPageFilterSkipBytes", |
77 | 404 | TUnit::BYTES, parquet_profile, 1); |
78 | 404 | selected_rows = |
79 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); |
80 | | // Keep every Parquet scan metric below the format node: profile consumers extract that |
81 | | // subtree and otherwise silently lose this counter even though filtering happened. |
82 | 404 | rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", |
83 | 404 | TUnit::UNIT, parquet_profile, 1); |
84 | 404 | total_batches = |
85 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); |
86 | 404 | dense_batches = |
87 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); |
88 | 404 | selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, |
89 | 404 | parquet_profile, 1); |
90 | 404 | empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", |
91 | 404 | TUnit::UNIT, parquet_profile, 1); |
92 | 404 | range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", |
93 | 404 | TUnit::UNIT, parquet_profile, 1); |
94 | 404 | reader_read_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderReadRows", TUnit::UNIT, |
95 | 404 | parquet_profile, 1); |
96 | 404 | reader_skip_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSkipRows", TUnit::UNIT, |
97 | 404 | parquet_profile, 1); |
98 | 404 | reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, |
99 | 404 | parquet_profile, 1); |
100 | 404 | level_only_read_time = |
101 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); |
102 | 404 | level_only_skip_time = |
103 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); |
104 | 404 | materialization_time = |
105 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); |
106 | 404 | variant_reconstruction_time = add_persistent_counter(profile, "VariantReconstructionTime", |
107 | 404 | TUnit::TIME_NS, parquet_profile); |
108 | 404 | variant_reconstructed_rows = add_persistent_counter(profile, "VariantReconstructedRows", |
109 | 404 | TUnit::UNIT, parquet_profile); |
110 | 404 | variant_direct_leaf_rows = |
111 | 404 | add_persistent_counter(profile, "VariantDirectLeafRows", TUnit::UNIT, parquet_profile); |
112 | 404 | variant_direct_leaf_path_misses = add_persistent_counter(profile, "VariantDirectLeafPathMisses", |
113 | 404 | TUnit::UNIT, parquet_profile); |
114 | 404 | variant_direct_leaf_residual_fallbacks = add_persistent_counter( |
115 | 404 | profile, "VariantDirectLeafResidualFallbacks", TUnit::UNIT, parquet_profile); |
116 | 404 | variant_direct_leaf_unsupported_fallbacks = add_persistent_counter( |
117 | 404 | profile, "VariantDirectLeafUnsupportedFallbacks", TUnit::UNIT, parquet_profile); |
118 | 404 | hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", |
119 | 404 | TUnit::UNIT, parquet_profile, 1); |
120 | 404 | hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", |
121 | 404 | TUnit::UNIT, parquet_profile, 1); |
122 | 404 | hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( |
123 | 404 | profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); |
124 | 404 | native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, |
125 | 404 | parquet_profile, 1); |
126 | 404 | native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", |
127 | 404 | TUnit::UNIT, parquet_profile, 1); |
128 | 404 | page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", |
129 | 404 | TUnit::UNIT, parquet_profile, 1); |
130 | 404 | nested_batches = |
131 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); |
132 | 404 | lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", |
133 | 404 | TUnit::UNIT, parquet_profile, 1); |
134 | | // Format-specific counters stay below ParquetReader so subtree consumers cannot silently |
135 | | // attribute them to a different file format initialized on the same RuntimeProfile. |
136 | 404 | filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, |
137 | 404 | parquet_profile, 1); |
138 | 404 | raw_rows_read = |
139 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); |
140 | 404 | column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); |
141 | 404 | parse_meta_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseMetaTime", parquet_profile, 1); |
142 | 404 | parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); |
143 | 404 | file_reader_create_time = |
144 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); |
145 | 404 | open_file_num = |
146 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); |
147 | 404 | page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, |
148 | 404 | parquet_profile, 1); |
149 | 404 | page_index_filter_time = |
150 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); |
151 | 404 | read_page_index_time = |
152 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexReadTime", parquet_profile, 1); |
153 | 404 | parse_page_index_time = |
154 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexParseTime", parquet_profile, 1); |
155 | 404 | expr_zonemap_unusable = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExprZoneMapUnusableEvals", |
156 | 404 | TUnit::UNIT, parquet_profile, 1); |
157 | 404 | in_zonemap_point_check = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapPointCheckCount", |
158 | 404 | TUnit::UNIT, parquet_profile, 1); |
159 | 404 | in_zonemap_range_only = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InZoneMapRangeOnlyCount", |
160 | 404 | TUnit::UNIT, parquet_profile, 1); |
161 | 404 | row_group_filter_time = |
162 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); |
163 | 404 | file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", |
164 | 404 | TUnit::UNIT, parquet_profile, 1); |
165 | 404 | file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, |
166 | 404 | parquet_profile, 1); |
167 | 404 | decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); |
168 | 404 | decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, |
169 | 404 | parquet_profile, 1); |
170 | 404 | page_read_counter = |
171 | 404 | ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageReadCount", TUnit::UNIT, parquet_profile, 1); |
172 | 404 | page_cache_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheWriteCount", |
173 | 404 | TUnit::UNIT, parquet_profile, 1); |
174 | 404 | page_cache_compressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
175 | 404 | profile, "PageCacheCompressedWriteCount", TUnit::UNIT, parquet_profile, 1); |
176 | 404 | page_cache_decompressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
177 | 404 | profile, "PageCacheDecompressedWriteCount", TUnit::UNIT, parquet_profile, 1); |
178 | 404 | page_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheHitCount", TUnit::UNIT, |
179 | 404 | parquet_profile, 1); |
180 | 404 | page_cache_missing_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCacheMissingCount", |
181 | 404 | TUnit::UNIT, parquet_profile, 1); |
182 | 404 | page_cache_compressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
183 | 404 | profile, "PageCacheCompressedHitCount", TUnit::UNIT, parquet_profile, 1); |
184 | 404 | page_cache_decompressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( |
185 | 404 | profile, "PageCacheDecompressedHitCount", TUnit::UNIT, parquet_profile, 1); |
186 | 404 | decode_header_time = |
187 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderDecodeTime", parquet_profile, 1); |
188 | 404 | read_page_header_time = |
189 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageHeaderReadTime", parquet_profile, 1); |
190 | 404 | decode_value_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeValueTime", parquet_profile, 1); |
191 | 404 | decode_dict_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeDictTime", parquet_profile, 1); |
192 | 404 | decode_level_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeLevelTime", parquet_profile, 1); |
193 | 404 | decode_null_map_time = |
194 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecodeNullMapTime", parquet_profile, 1); |
195 | 404 | skip_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SkipPageHeaderNum", TUnit::UNIT, |
196 | 404 | parquet_profile, 1); |
197 | 404 | parse_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ParsePageHeaderNum", TUnit::UNIT, |
198 | 404 | parquet_profile, 1); |
199 | 404 | predicate_filter_time = |
200 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); |
201 | 404 | predicate_compaction_time = |
202 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); |
203 | 404 | predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", |
204 | 404 | TUnit::BYTES, parquet_profile, 1); |
205 | 404 | predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", |
206 | 404 | TUnit::UNIT, parquet_profile, 1); |
207 | 404 | predicate_alignment_columns = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateAlignmentColumns", |
208 | 404 | TUnit::UNIT, parquet_profile, 1); |
209 | 404 | fixed_width_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( |
210 | 404 | profile, "FixedWidthPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); |
211 | 404 | fixed_width_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
212 | 404 | profile, "FixedWidthPredicateDirectRows", TUnit::UNIT, parquet_profile, 1); |
213 | 404 | raw_value_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( |
214 | 404 | profile, "RawValuePredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); |
215 | 404 | raw_value_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
216 | 404 | profile, "RawValuePredicateDirectRows", TUnit::UNIT, parquet_profile, 1); |
217 | 404 | typed_runtime_filter_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( |
218 | 404 | profile, "TypedRuntimeFilterDirectBatches", TUnit::UNIT, parquet_profile, 1); |
219 | 404 | typed_runtime_filter_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
220 | 404 | profile, "TypedRuntimeFilterDirectRows", TUnit::UNIT, parquet_profile, 1); |
221 | 404 | dictionary_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( |
222 | 404 | profile, "DictionaryPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); |
223 | 404 | dictionary_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
224 | 404 | profile, "DictionaryPredicateDirectRows", TUnit::UNIT, parquet_profile, 1); |
225 | 404 | dictionary_predicate_projected_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
226 | 404 | profile, "DictionaryPredicateProjectedRows", TUnit::UNIT, parquet_profile, 1); |
227 | 404 | dictionary_predicate_fused_projected_rows = ADD_CHILD_COUNTER_WITH_LEVEL( |
228 | 404 | profile, "DictionaryPredicateFusedProjectedRows", TUnit::UNIT, parquet_profile, 1); |
229 | 404 | dict_filter_rewrite_time = |
230 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); |
231 | 404 | dict_filter_expr_rewrite_time = |
232 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterExprRewriteTime", parquet_profile, 1); |
233 | 404 | dict_filter_read_dict_time = |
234 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterReadDictTime", parquet_profile, 1); |
235 | 404 | dict_filter_build_time = |
236 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterBuildTime", parquet_profile, 1); |
237 | 404 | dict_filter_candidate_columns = ADD_CHILD_COUNTER_WITH_LEVEL( |
238 | 404 | profile, "DictFilterCandidateColumns", TUnit::UNIT, parquet_profile, 1); |
239 | 404 | dict_filter_columns = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterColumns", TUnit::UNIT, |
240 | 404 | parquet_profile, 1); |
241 | 404 | dict_filter_typed_compare_columns = ADD_CHILD_COUNTER_WITH_LEVEL( |
242 | 404 | profile, "DictFilterTypedCompareColumns", TUnit::UNIT, parquet_profile, 1); |
243 | 404 | dict_filter_string_compare_columns = ADD_CHILD_COUNTER_WITH_LEVEL( |
244 | 404 | profile, "DictFilterStringCompareColumns", TUnit::UNIT, parquet_profile, 1); |
245 | 404 | dict_filter_vectorized_runtime_filter_columns = ADD_CHILD_COUNTER_WITH_LEVEL( |
246 | 404 | profile, "DictFilterVectorizedRuntimeFilterColumns", TUnit::UNIT, parquet_profile, 1); |
247 | 404 | dict_filter_unsupported_columns = ADD_CHILD_COUNTER_WITH_LEVEL( |
248 | 404 | profile, "DictFilterUnsupportedColumns", TUnit::UNIT, parquet_profile, 1); |
249 | 404 | dict_filter_read_failures = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterReadFailures", |
250 | 404 | TUnit::UNIT, parquet_profile, 1); |
251 | 404 | rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", |
252 | 404 | TUnit::UNIT, parquet_profile, 1); |
253 | 404 | bloom_filter_read_time = |
254 | 404 | ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); |
255 | 404 | } |
256 | | |
257 | 194 | void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_stats) const { |
258 | 194 | COUNTER_UPDATE(filtered_row_groups, |
259 | 194 | pruning_stats.total_row_groups - pruning_stats.selected_row_groups); |
260 | 194 | COUNTER_UPDATE(filtered_row_groups_by_min_max, pruning_stats.filtered_row_groups_by_statistics); |
261 | 194 | COUNTER_UPDATE(filtered_row_groups_by_dictionary, |
262 | 194 | pruning_stats.filtered_row_groups_by_dictionary); |
263 | 194 | COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, |
264 | 194 | pruning_stats.filtered_row_groups_by_bloom_filter); |
265 | 194 | COUNTER_UPDATE(filtered_row_groups_by_page_index, |
266 | 194 | pruning_stats.filtered_row_groups_by_page_index); |
267 | 194 | COUNTER_UPDATE(to_read_row_groups, pruning_stats.selected_row_groups); |
268 | 194 | COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); |
269 | 194 | COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); |
270 | 194 | COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); |
271 | 194 | COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); |
272 | 194 | COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); |
273 | 194 | COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); |
274 | 194 | COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); |
275 | 194 | COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); |
276 | 194 | COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); |
277 | 194 | COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); |
278 | 194 | COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); |
279 | 194 | COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); |
280 | 194 | COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); |
281 | 194 | COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); |
282 | 194 | } |
283 | | |
284 | | void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, |
285 | 213 | bool selected) const { |
286 | 213 | const int64_t filtered = selected ? 0 : 1; |
287 | 213 | COUNTER_UPDATE(filtered_row_groups, filtered); |
288 | 213 | COUNTER_UPDATE(filtered_row_groups_by_min_max, pruning_stats.filtered_row_groups_by_statistics); |
289 | 213 | COUNTER_UPDATE(filtered_row_groups_by_dictionary, |
290 | 213 | pruning_stats.filtered_row_groups_by_dictionary); |
291 | 213 | COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, |
292 | 213 | pruning_stats.filtered_row_groups_by_bloom_filter); |
293 | 213 | COUNTER_UPDATE(filtered_row_groups_by_page_index, |
294 | 213 | pruning_stats.filtered_row_groups_by_page_index); |
295 | | // Initial footer planning counts every surviving candidate as readable. Lazy probes correct |
296 | | // that estimate only when a later dictionary, Bloom, or page-index check removes the group. |
297 | 213 | COUNTER_UPDATE(to_read_row_groups, -filtered); |
298 | 213 | COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); |
299 | 213 | COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); |
300 | 213 | COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); |
301 | 213 | COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); |
302 | 213 | COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); |
303 | 213 | COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); |
304 | 213 | COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); |
305 | 213 | COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); |
306 | 213 | COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); |
307 | 213 | COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); |
308 | 213 | COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); |
309 | 213 | COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); |
310 | 213 | COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); |
311 | 213 | } |
312 | | |
313 | 307 | ParquetPageSkipProfile ParquetProfile::page_skip_profile() const { |
314 | 307 | return { |
315 | 307 | .skipped_pages = pages_skipped_by_data_page_filter, |
316 | 307 | .skipped_bytes = data_page_filter_skip_bytes, |
317 | 307 | }; |
318 | 307 | } |
319 | | |
320 | 651 | ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { |
321 | 651 | return { |
322 | 651 | .reader_read_rows = reader_read_rows, |
323 | 651 | .reader_skip_rows = reader_skip_rows, |
324 | 651 | .reader_select_rows = reader_select_rows, |
325 | 651 | .level_only_read_time = level_only_read_time, |
326 | 651 | .level_only_skip_time = level_only_skip_time, |
327 | 651 | .materialization_time = materialization_time, |
328 | 651 | .variant_reconstruction_time = variant_reconstruction_time, |
329 | 651 | .variant_reconstructed_rows = variant_reconstructed_rows, |
330 | 651 | .variant_direct_leaf_rows = variant_direct_leaf_rows, |
331 | 651 | .variant_direct_leaf_path_misses = variant_direct_leaf_path_misses, |
332 | 651 | .variant_direct_leaf_residual_fallbacks = variant_direct_leaf_residual_fallbacks, |
333 | 651 | .variant_direct_leaf_unsupported_fallbacks = variant_direct_leaf_unsupported_fallbacks, |
334 | 651 | .hybrid_selection_batches = hybrid_selection_batches, |
335 | 651 | .hybrid_selection_ranges = hybrid_selection_ranges, |
336 | 651 | .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, |
337 | 651 | .dictionary_predicate_fused_projected_rows = dictionary_predicate_fused_projected_rows, |
338 | 651 | .decompress_time = decompress_time, |
339 | 651 | .decompress_count = decompress_cnt, |
340 | 651 | .decode_header_time = decode_header_time, |
341 | 651 | .decode_value_time = decode_value_time, |
342 | 651 | .decode_dictionary_time = decode_dict_time, |
343 | 651 | .decode_level_time = decode_level_time, |
344 | 651 | .decode_null_map_time = decode_null_map_time, |
345 | 651 | .page_index_read_calls = page_index_read_calls, |
346 | 651 | .skip_page_header_count = skip_page_header_num, |
347 | 651 | .parse_page_header_count = parse_page_header_num, |
348 | 651 | .read_page_header_time = read_page_header_time, |
349 | 651 | .page_read_count = page_read_counter, |
350 | 651 | .page_cache_write_count = page_cache_write_counter, |
351 | 651 | .page_cache_compressed_write_count = page_cache_compressed_write_counter, |
352 | 651 | .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, |
353 | 651 | .page_cache_hit_count = page_cache_hit_counter, |
354 | 651 | .page_cache_miss_count = page_cache_missing_counter, |
355 | 651 | .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, |
356 | 651 | .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, |
357 | 651 | .native_read_calls = native_read_calls, |
358 | 651 | .native_page_fragments = native_page_fragments, |
359 | 651 | .page_crossing_batches = page_crossing_batches, |
360 | 651 | .nested_batches = nested_batches, |
361 | 651 | }; |
362 | 651 | } |
363 | | |
364 | 314 | ParquetScanProfile ParquetProfile::scan_profile() const { |
365 | 314 | return { |
366 | 314 | .raw_rows_read = raw_rows_read, |
367 | 314 | .selected_rows = selected_rows, |
368 | 314 | .rows_filtered_by_conjunct = rows_filtered_by_conjunct, |
369 | 314 | .lazy_read_filtered_rows = lazy_read_filtered_rows, |
370 | 314 | .total_batches = total_batches, |
371 | 314 | .dense_batches = dense_batches, |
372 | 314 | .selected_batches = selected_batches, |
373 | 314 | .empty_selection_batches = empty_selection_batches, |
374 | 314 | .range_gap_skipped_rows = range_gap_skipped_rows, |
375 | 314 | .column_read_time = column_read_time, |
376 | 314 | .predicate_filter_time = predicate_filter_time, |
377 | 314 | .predicate_compaction_time = predicate_compaction_time, |
378 | 314 | .predicate_compaction_bytes = predicate_compaction_bytes, |
379 | 314 | .predicate_compaction_count = predicate_compaction_count, |
380 | 314 | .predicate_alignment_columns = predicate_alignment_columns, |
381 | 314 | .fixed_width_predicate_direct_batches = fixed_width_predicate_direct_batches, |
382 | 314 | .fixed_width_predicate_direct_rows = fixed_width_predicate_direct_rows, |
383 | 314 | .raw_value_predicate_direct_batches = raw_value_predicate_direct_batches, |
384 | 314 | .raw_value_predicate_direct_rows = raw_value_predicate_direct_rows, |
385 | 314 | .typed_runtime_filter_direct_batches = typed_runtime_filter_direct_batches, |
386 | 314 | .typed_runtime_filter_direct_rows = typed_runtime_filter_direct_rows, |
387 | 314 | .dictionary_predicate_direct_batches = dictionary_predicate_direct_batches, |
388 | 314 | .dictionary_predicate_direct_rows = dictionary_predicate_direct_rows, |
389 | 314 | .dictionary_predicate_projected_rows = dictionary_predicate_projected_rows, |
390 | 314 | .dict_filter_rewrite_time = dict_filter_rewrite_time, |
391 | 314 | .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, |
392 | 314 | .dict_filter_read_dict_time = dict_filter_read_dict_time, |
393 | 314 | .dict_filter_build_time = dict_filter_build_time, |
394 | 314 | .dict_filter_candidate_columns = dict_filter_candidate_columns, |
395 | 314 | .dict_filter_columns = dict_filter_columns, |
396 | 314 | .dict_filter_typed_compare_columns = dict_filter_typed_compare_columns, |
397 | 314 | .dict_filter_string_compare_columns = dict_filter_string_compare_columns, |
398 | 314 | .dict_filter_vectorized_runtime_filter_columns = |
399 | 314 | dict_filter_vectorized_runtime_filter_columns, |
400 | 314 | .dict_filter_unsupported_columns = dict_filter_unsupported_columns, |
401 | 314 | .dict_filter_read_failures = dict_filter_read_failures, |
402 | 314 | .rows_filtered_by_dict_filter = rows_filtered_by_dict_filter, |
403 | 314 | .column_reader_profile = column_reader_profile(), |
404 | 314 | }; |
405 | 314 | } |
406 | | |
407 | | } // namespace doris::format::parquet |