be/benchmark/parquet/parquet_benchmark_scenarios.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <cstddef> |
21 | | #include <cstdint> |
22 | | #include <set> |
23 | | #include <string> |
24 | | #include <tuple> |
25 | | #include <vector> |
26 | | |
27 | | namespace doris::parquet_benchmark { |
28 | | |
29 | | enum class Encoding { |
30 | | PLAIN, |
31 | | DICTIONARY, |
32 | | BYTE_STREAM_SPLIT, |
33 | | DELTA_BINARY_PACKED, |
34 | | DELTA_LENGTH_BYTE_ARRAY, |
35 | | DELTA_BYTE_ARRAY |
36 | | }; |
37 | | enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY }; |
38 | | enum class Pattern { CLUSTERED, ALTERNATING }; |
39 | | enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED }; |
40 | | enum class ReaderOperation { OPEN_TO_FIRST_BLOCK, FULL_SCAN, PREDICATE_SCAN, LIMIT_1, LIMIT_1000 }; |
41 | | |
42 | | struct DecoderScenario { |
43 | | Encoding encoding; |
44 | | ValueType value_type; |
45 | | }; |
46 | | |
47 | | struct ReaderScenario { |
48 | | ReaderOperation operation; |
49 | | Encoding encoding; |
50 | | int null_percent; |
51 | | Pattern null_pattern; |
52 | | int selectivity_percent; |
53 | | Projection projection; |
54 | | int schema_width; |
55 | | int predicate_position; |
56 | | }; |
57 | | |
58 | | struct SelectionRange { |
59 | | size_t first; |
60 | | size_t count; |
61 | | }; |
62 | | |
63 | | struct SelectionPlan { |
64 | | size_t total_rows = 0; |
65 | | size_t selected_rows = 0; |
66 | | std::vector<SelectionRange> ranges; |
67 | | }; |
68 | | |
69 | 1 | inline std::vector<DecoderScenario> decoder_scenarios() { |
70 | 1 | return { |
71 | 1 | {Encoding::PLAIN, ValueType::INT32}, |
72 | 1 | {Encoding::PLAIN, ValueType::INT64}, |
73 | 1 | {Encoding::PLAIN, ValueType::FLOAT}, |
74 | 1 | {Encoding::PLAIN, ValueType::DOUBLE}, |
75 | 1 | {Encoding::PLAIN, ValueType::BYTE_ARRAY}, |
76 | 1 | {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, |
77 | 1 | {Encoding::DICTIONARY, ValueType::INT32}, |
78 | 1 | {Encoding::DICTIONARY, ValueType::INT64}, |
79 | 1 | {Encoding::DICTIONARY, ValueType::FLOAT}, |
80 | 1 | {Encoding::DICTIONARY, ValueType::DOUBLE}, |
81 | 1 | {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, |
82 | 1 | {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, |
83 | 1 | {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, |
84 | 1 | {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, |
85 | 1 | {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, |
86 | 1 | {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, |
87 | 1 | {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, |
88 | 1 | {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, |
89 | 1 | {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, |
90 | 1 | }; |
91 | 1 | } |
92 | | |
93 | 2 | inline std::vector<ReaderScenario> reader_scenarios() { |
94 | 2 | std::vector<ReaderScenario> scenarios; |
95 | 2 | std::set<std::tuple<ReaderOperation, Encoding, int, Pattern, int, Projection, int, int>> seen; |
96 | 282 | const auto add = [&](ReaderScenario scenario) { |
97 | 282 | const auto key = std::make_tuple(scenario.operation, scenario.encoding, |
98 | 282 | scenario.null_percent, scenario.null_pattern, |
99 | 282 | scenario.selectivity_percent, scenario.projection, |
100 | 282 | scenario.schema_width, scenario.predicate_position); |
101 | 282 | if (seen.insert(key).second) { |
102 | 274 | scenarios.push_back(scenario); |
103 | 274 | } |
104 | 282 | }; |
105 | | |
106 | 2 | const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN, |
107 | 2 | .encoding = Encoding::PLAIN, |
108 | 2 | .null_percent = 10, |
109 | 2 | .null_pattern = Pattern::ALTERNATING, |
110 | 2 | .selectivity_percent = 10, |
111 | 2 | .projection = Projection::PREDICATE_PROJECTED, |
112 | 2 | .schema_width = 32, |
113 | 2 | .predicate_position = 0}; |
114 | 2 | for (const auto operation : |
115 | 2 | {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, |
116 | 10 | ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { |
117 | 10 | auto scenario = baseline; |
118 | 10 | scenario.operation = operation; |
119 | 10 | add(scenario); |
120 | 10 | } |
121 | 2 | for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, |
122 | 8 | Encoding::DELTA_BINARY_PACKED}) { |
123 | 8 | auto scenario = baseline; |
124 | 8 | scenario.encoding = encoding; |
125 | 8 | add(scenario); |
126 | 8 | scenario.operation = ReaderOperation::PREDICATE_SCAN; |
127 | 8 | add(scenario); |
128 | 8 | } |
129 | 8 | for (const int width : {4, 32, 128, 512}) { |
130 | 16 | for (const int predicate_position : {0, width - 1}) { |
131 | 16 | auto scenario = baseline; |
132 | 16 | scenario.operation = ReaderOperation::PREDICATE_SCAN; |
133 | 16 | scenario.schema_width = width; |
134 | 16 | scenario.predicate_position = predicate_position; |
135 | 16 | add(scenario); |
136 | 16 | } |
137 | 8 | } |
138 | 10 | for (const int null_percent : {0, 1, 10, 50, 90}) { |
139 | 20 | for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { |
140 | 120 | for (const int selectivity : {0, 1, 10, 50, 90, 100}) { |
141 | 120 | for (const auto projection : |
142 | 240 | {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { |
143 | 240 | auto scenario = baseline; |
144 | 240 | scenario.operation = ReaderOperation::PREDICATE_SCAN; |
145 | 240 | scenario.null_percent = null_percent; |
146 | 240 | scenario.null_pattern = pattern; |
147 | 240 | scenario.selectivity_percent = selectivity; |
148 | 240 | scenario.projection = projection; |
149 | 240 | add(scenario); |
150 | 240 | } |
151 | 120 | } |
152 | 20 | } |
153 | 10 | } |
154 | 2 | return scenarios; |
155 | 2 | } |
156 | | |
157 | | inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent, |
158 | 4 | Pattern pattern) { |
159 | 4 | SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}}; |
160 | 4 | if (total_rows == 0 || selectivity_percent <= 0) { |
161 | 1 | return plan; |
162 | 1 | } |
163 | 3 | if (selectivity_percent >= 100) { |
164 | 1 | plan.selected_rows = total_rows; |
165 | 1 | plan.ranges.push_back({.first = 0, .count = total_rows}); |
166 | 1 | return plan; |
167 | 1 | } |
168 | 2 | plan.selected_rows = total_rows * static_cast<size_t>(selectivity_percent) / 100; |
169 | 2 | if (plan.selected_rows == 0) { |
170 | 0 | plan.selected_rows = 1; |
171 | 0 | } |
172 | 2 | if (pattern == Pattern::CLUSTERED) { |
173 | 1 | plan.ranges.push_back({.first = 0, .count = plan.selected_rows}); |
174 | 1 | return plan; |
175 | 1 | } |
176 | | |
177 | | // Evenly spaced rows deliberately maximize the number of physical ranges. This is the |
178 | | // adversarial sparse shape that exposes per-run decoder and cursor overhead. |
179 | 101 | for (size_t selected = 0; selected < plan.selected_rows; ++selected) { |
180 | 100 | const size_t row = selected * total_rows / plan.selected_rows; |
181 | 100 | if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) { |
182 | 0 | ++plan.ranges.back().count; |
183 | 100 | } else { |
184 | 100 | plan.ranges.push_back({.first = row, .count = 1}); |
185 | 100 | } |
186 | 100 | } |
187 | 1 | return plan; |
188 | 2 | } |
189 | | |
190 | 0 | inline std::string to_string(Encoding value) { |
191 | 0 | switch (value) { |
192 | 0 | case Encoding::PLAIN: |
193 | 0 | return "plain"; |
194 | 0 | case Encoding::DICTIONARY: |
195 | 0 | return "dictionary"; |
196 | 0 | case Encoding::BYTE_STREAM_SPLIT: |
197 | 0 | return "byte_stream_split"; |
198 | 0 | case Encoding::DELTA_BINARY_PACKED: |
199 | 0 | return "delta_binary_packed"; |
200 | 0 | case Encoding::DELTA_LENGTH_BYTE_ARRAY: |
201 | 0 | return "delta_length_byte_array"; |
202 | 0 | case Encoding::DELTA_BYTE_ARRAY: |
203 | 0 | return "delta_byte_array"; |
204 | 0 | } |
205 | 0 | return "unknown"; |
206 | 0 | } |
207 | | |
208 | 0 | inline std::string to_string(ValueType value) { |
209 | 0 | switch (value) { |
210 | 0 | case ValueType::INT32: |
211 | 0 | return "int32"; |
212 | 0 | case ValueType::INT64: |
213 | 0 | return "int64"; |
214 | 0 | case ValueType::FLOAT: |
215 | 0 | return "float"; |
216 | 0 | case ValueType::DOUBLE: |
217 | 0 | return "double"; |
218 | 0 | case ValueType::BYTE_ARRAY: |
219 | 0 | return "byte_array"; |
220 | 0 | case ValueType::FIXED_LEN_BYTE_ARRAY: |
221 | 0 | return "fixed_len_byte_array"; |
222 | 0 | } |
223 | 0 | return "unknown"; |
224 | 0 | } |
225 | | |
226 | 0 | inline std::string to_string(Pattern value) { |
227 | 0 | return value == Pattern::CLUSTERED ? "clustered" : "alternating"; |
228 | 0 | } |
229 | | |
230 | 0 | inline std::string to_string(Projection value) { |
231 | 0 | return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected"; |
232 | 0 | } |
233 | | |
234 | 0 | inline std::string to_string(ReaderOperation value) { |
235 | 0 | switch (value) { |
236 | 0 | case ReaderOperation::OPEN_TO_FIRST_BLOCK: |
237 | 0 | return "open_to_first_block"; |
238 | 0 | case ReaderOperation::FULL_SCAN: |
239 | 0 | return "full_scan"; |
240 | 0 | case ReaderOperation::PREDICATE_SCAN: |
241 | 0 | return "predicate_scan"; |
242 | 0 | case ReaderOperation::LIMIT_1: |
243 | 0 | return "limit_1"; |
244 | 0 | case ReaderOperation::LIMIT_1000: |
245 | 0 | return "limit_1000"; |
246 | 0 | } |
247 | 0 | return "unknown"; |
248 | 0 | } |
249 | | |
250 | | } // namespace doris::parquet_benchmark |