be/benchmark/parquet/parquet_benchmark_scenarios.h

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <cstddef>
#include <cstdint>
#include <set>
#include <string>
#include <tuple>
#include <vector>

namespace doris::parquet_benchmark {

enum class Encoding {
    PLAIN,
    DICTIONARY,
    BYTE_STREAM_SPLIT,
    DELTA_BINARY_PACKED,
    DELTA_LENGTH_BYTE_ARRAY,
    DELTA_BYTE_ARRAY
};
enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY };
enum class Pattern { CLUSTERED, ALTERNATING };
enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED };
enum class ReaderOperation {
    OPEN_TO_FIRST_BLOCK,
    FULL_SCAN,
    PREDICATE_SCAN,
    COMPLEX_RESIDUAL_SCAN,
    LIMIT_1,
    LIMIT_1000
};

struct DecoderScenario {
    Encoding encoding;
    ValueType value_type;
};

struct ReaderScenario {
    ReaderOperation operation;
    Encoding encoding;
    int null_percent;
    Pattern null_pattern;
    int selectivity_percent;
    Projection projection;
    int schema_width;
    int predicate_position;
};

struct SelectionRange {
    size_t first;
    size_t count;
};

struct SelectionPlan {
    size_t total_rows = 0;
    size_t selected_rows = 0;
    std::vector<SelectionRange> ranges;
};

inline std::vector<DecoderScenario> decoder_scenarios() {
    return {
            {Encoding::PLAIN, ValueType::INT32},
            {Encoding::PLAIN, ValueType::INT64},
            {Encoding::PLAIN, ValueType::FLOAT},
            {Encoding::PLAIN, ValueType::DOUBLE},
            {Encoding::PLAIN, ValueType::BYTE_ARRAY},
            {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY},
            {Encoding::DICTIONARY, ValueType::INT32},
            {Encoding::DICTIONARY, ValueType::INT64},
            {Encoding::DICTIONARY, ValueType::FLOAT},
            {Encoding::DICTIONARY, ValueType::DOUBLE},
            {Encoding::DICTIONARY, ValueType::BYTE_ARRAY},
            {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY},
            {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT},
            {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE},
            {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY},
            {Encoding::DELTA_BINARY_PACKED, ValueType::INT32},
            {Encoding::DELTA_BINARY_PACKED, ValueType::INT64},
            {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY},
            {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY},
    };
}

inline std::vector<ReaderScenario> reader_scenarios() {
    std::vector<ReaderScenario> scenarios;
    std::set<std::tuple<ReaderOperation, Encoding, int, Pattern, int, Projection, int, int>> seen;
    const auto add = [&](ReaderScenario scenario) {
        const auto key = std::make_tuple(scenario.operation, scenario.encoding,
                                         scenario.null_percent, scenario.null_pattern,
                                         scenario.selectivity_percent, scenario.projection,
                                         scenario.schema_width, scenario.predicate_position);
        if (seen.insert(key).second) {
            scenarios.push_back(scenario);
        }
    };

    const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN,
                                   .encoding = Encoding::PLAIN,
                                   .null_percent = 10,
                                   .null_pattern = Pattern::ALTERNATING,
                                   .selectivity_percent = 10,
                                   .projection = Projection::PREDICATE_PROJECTED,
                                   .schema_width = 32,
                                   .predicate_position = 0};
    for (const auto operation :
         {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN,
          ReaderOperation::PREDICATE_SCAN, ReaderOperation::COMPLEX_RESIDUAL_SCAN,
          ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) {
        auto scenario = baseline;
        scenario.operation = operation;
        add(scenario);
    }
    for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT,
                                Encoding::DELTA_BINARY_PACKED}) {
        auto scenario = baseline;
        scenario.encoding = encoding;
        add(scenario);
        scenario.operation = ReaderOperation::PREDICATE_SCAN;
        add(scenario);
    }
    for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) {
        for (const int selectivity : {1, 10, 50, 90}) {
            for (const auto projection :
                 {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) {
                auto scenario = baseline;
                scenario.operation = ReaderOperation::PREDICATE_SCAN;
                scenario.encoding = encoding;
                scenario.selectivity_percent = selectivity;
                scenario.projection = projection;
                add(scenario);
            }
        }
    }
    for (const int width : {4, 32, 128, 512}) {
        for (const int predicate_position : {0, width - 1}) {
            auto scenario = baseline;
            scenario.operation = ReaderOperation::PREDICATE_SCAN;
            scenario.schema_width = width;
            scenario.predicate_position = predicate_position;
            add(scenario);
        }
    }
    for (const int null_percent : {0, 1, 10, 50, 90}) {
        for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) {
            for (const int selectivity : {0, 1, 10, 50, 90, 100}) {
                for (const auto projection :
                     {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) {
                    auto scenario = baseline;
                    scenario.operation = ReaderOperation::PREDICATE_SCAN;
                    scenario.null_percent = null_percent;
                    scenario.null_pattern = pattern;
                    scenario.selectivity_percent = selectivity;
                    scenario.projection = projection;
                    add(scenario);
                }
            }
        }
    }
    return scenarios;
}

inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent,
                                         Pattern pattern) {
    SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}};
    if (total_rows == 0 || selectivity_percent <= 0) {
        return plan;
    }
    if (selectivity_percent >= 100) {
        plan.selected_rows = total_rows;
        plan.ranges.push_back({.first = 0, .count = total_rows});
        return plan;
    }
    plan.selected_rows = total_rows * static_cast<size_t>(selectivity_percent) / 100;
    if (plan.selected_rows == 0) {
        plan.selected_rows = 1;
    }
    if (pattern == Pattern::CLUSTERED) {
        plan.ranges.push_back({.first = 0, .count = plan.selected_rows});
        return plan;
    }

    // Evenly spaced rows deliberately maximize the number of physical ranges. This is the
    // adversarial sparse shape that exposes per-run decoder and cursor overhead.
    for (size_t selected = 0; selected < plan.selected_rows; ++selected) {
        const size_t row = selected * total_rows / plan.selected_rows;
        if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) {
            ++plan.ranges.back().count;
        } else {
            plan.ranges.push_back({.first = row, .count = 1});
        }
    }
    return plan;
}

inline std::string to_string(Encoding value) {
    switch (value) {
    case Encoding::PLAIN:
        return "plain";
    case Encoding::DICTIONARY:
        return "dictionary";
    case Encoding::BYTE_STREAM_SPLIT:
        return "byte_stream_split";
    case Encoding::DELTA_BINARY_PACKED:
        return "delta_binary_packed";
    case Encoding::DELTA_LENGTH_BYTE_ARRAY:
        return "delta_length_byte_array";
    case Encoding::DELTA_BYTE_ARRAY:
        return "delta_byte_array";
    }
    return "unknown";
}

inline std::string to_string(ValueType value) {
    switch (value) {
    case ValueType::INT32:
        return "int32";
    case ValueType::INT64:
        return "int64";
    case ValueType::FLOAT:
        return "float";
    case ValueType::DOUBLE:
        return "double";
    case ValueType::BYTE_ARRAY:
        return "byte_array";
    case ValueType::FIXED_LEN_BYTE_ARRAY:
        return "fixed_len_byte_array";
    }
    return "unknown";
}

inline std::string to_string(Pattern value) {
    return value == Pattern::CLUSTERED ? "clustered" : "alternating";
}

inline std::string to_string(Projection value) {
    return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected";
}

inline std::string to_string(ReaderOperation value) {
    switch (value) {
    case ReaderOperation::OPEN_TO_FIRST_BLOCK:
        return "open_to_first_block";
    case ReaderOperation::FULL_SCAN:
        return "full_scan";
    case ReaderOperation::PREDICATE_SCAN:
        return "predicate_scan";
    case ReaderOperation::COMPLEX_RESIDUAL_SCAN:
        return "complex_residual_scan";
    case ReaderOperation::LIMIT_1:
        return "limit_1";
    case ReaderOperation::LIMIT_1000:
        return "limit_1000";
    }
    return "unknown";
}

inline std::string reader_scenario_name(const ReaderScenario& scenario) {
    return to_string(scenario.operation) + "/" + to_string(scenario.encoding) + "/null_" +
           std::to_string(scenario.null_percent) + "/" + to_string(scenario.null_pattern) +
           "/sel_" + std::to_string(scenario.selectivity_percent) + "/" +
           to_string(scenario.projection) + "/width_" + std::to_string(scenario.schema_width) +
           "/predicate_" + std::to_string(scenario.predicate_position);
}

} // namespace doris::parquet_benchmark

Coverage Report

Created: 2026-07-24 01:23

Line	Count	Source
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17
18		#pragma once
19
20		#include <cstddef>
21		#include <cstdint>
22		#include <set>
23		#include <string>
24		#include <tuple>
25		#include <vector>
26
27		namespace doris::parquet_benchmark {
28
29		enum class Encoding {
30		PLAIN,
31		DICTIONARY,
32		BYTE_STREAM_SPLIT,
33		DELTA_BINARY_PACKED,
34		DELTA_LENGTH_BYTE_ARRAY,
35		DELTA_BYTE_ARRAY
36		};
37		enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY };
38		enum class Pattern { CLUSTERED, ALTERNATING };
39		enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED };
40		enum class ReaderOperation {
41		OPEN_TO_FIRST_BLOCK,
42		FULL_SCAN,
43		PREDICATE_SCAN,
44		COMPLEX_RESIDUAL_SCAN,
45		LIMIT_1,
46		LIMIT_1000
47		};
48
49		struct DecoderScenario {
50		Encoding encoding;
51		ValueType value_type;
52		};
53
54		struct ReaderScenario {
55		ReaderOperation operation;
56		Encoding encoding;
57		int null_percent;
58		Pattern null_pattern;
59		int selectivity_percent;
60		Projection projection;
61		int schema_width;
62		int predicate_position;
63		};
64
65		struct SelectionRange {
66		size_t first;
67		size_t count;
68		};
69
70		struct SelectionPlan {
71		size_t total_rows = 0;
72		size_t selected_rows = 0;
73		std::vector<SelectionRange> ranges;
74		};
75
76	1	inline std::vector<DecoderScenario> decoder_scenarios() {
77	1	return {
78	1	{Encoding::PLAIN, ValueType::INT32},
79	1	{Encoding::PLAIN, ValueType::INT64},
80	1	{Encoding::PLAIN, ValueType::FLOAT},
81	1	{Encoding::PLAIN, ValueType::DOUBLE},
82	1	{Encoding::PLAIN, ValueType::BYTE_ARRAY},
83	1	{Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY},
84	1	{Encoding::DICTIONARY, ValueType::INT32},
85	1	{Encoding::DICTIONARY, ValueType::INT64},
86	1	{Encoding::DICTIONARY, ValueType::FLOAT},
87	1	{Encoding::DICTIONARY, ValueType::DOUBLE},
88	1	{Encoding::DICTIONARY, ValueType::BYTE_ARRAY},
89	1	{Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY},
90	1	{Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT},
91	1	{Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE},
92	1	{Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY},
93	1	{Encoding::DELTA_BINARY_PACKED, ValueType::INT32},
94	1	{Encoding::DELTA_BINARY_PACKED, ValueType::INT64},
95	1	{Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY},
96	1	{Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY},
97	1	};
98	1	}
99
100	5	inline std::vector<ReaderScenario> reader_scenarios() {
101	5	std::vector<ReaderScenario> scenarios;
102	5	std::set<std::tuple<ReaderOperation, Encoding, int, Pattern, int, Projection, int, int>> seen;
103	790	const auto add = [&](ReaderScenario scenario) {
104	790	const auto key = std::make_tuple(scenario.operation, scenario.encoding,
105	790	scenario.null_percent, scenario.null_pattern,
106	790	scenario.selectivity_percent, scenario.projection,
107	790	scenario.schema_width, scenario.predicate_position);
108	790	if (seen.insert(key).second) {
109	760	scenarios.push_back(scenario);
110	760	}
111	790	};
112
113	5	const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN,
114	5	.encoding = Encoding::PLAIN,
115	5	.null_percent = 10,
116	5	.null_pattern = Pattern::ALTERNATING,
117	5	.selectivity_percent = 10,
118	5	.projection = Projection::PREDICATE_PROJECTED,
119	5	.schema_width = 32,
120	5	.predicate_position = 0};
121	5	for (const auto operation :
122	5	{ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN,
123	5	ReaderOperation::PREDICATE_SCAN, ReaderOperation::COMPLEX_RESIDUAL_SCAN,
124	30	ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) {
125	30	auto scenario = baseline;
126	30	scenario.operation = operation;
127	30	add(scenario);
128	30	}
129	5	for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT,
130	20	Encoding::DELTA_BINARY_PACKED}) {
131	20	auto scenario = baseline;
132	20	scenario.encoding = encoding;
133	20	add(scenario);
134	20	scenario.operation = ReaderOperation::PREDICATE_SCAN;
135	20	add(scenario);
136	20	}
137	10	for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) {
138	40	for (const int selectivity : {1, 10, 50, 90}) {
139	40	for (const auto projection :
140	80	{Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) {
141	80	auto scenario = baseline;
142	80	scenario.operation = ReaderOperation::PREDICATE_SCAN;
143	80	scenario.encoding = encoding;
144	80	scenario.selectivity_percent = selectivity;
145	80	scenario.projection = projection;
146	80	add(scenario);
147	80	}
148	40	}
149	10	}
150	20	for (const int width : {4, 32, 128, 512}) {
151	40	for (const int predicate_position : {0, width - 1}) {
152	40	auto scenario = baseline;
153	40	scenario.operation = ReaderOperation::PREDICATE_SCAN;
154	40	scenario.schema_width = width;
155	40	scenario.predicate_position = predicate_position;
156	40	add(scenario);
157	40	}
158	20	}
159	25	for (const int null_percent : {0, 1, 10, 50, 90}) {
160	50	for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) {
161	300	for (const int selectivity : {0, 1, 10, 50, 90, 100}) {
162	300	for (const auto projection :
163	600	{Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) {
164	600	auto scenario = baseline;
165	600	scenario.operation = ReaderOperation::PREDICATE_SCAN;
166	600	scenario.null_percent = null_percent;
167	600	scenario.null_pattern = pattern;
168	600	scenario.selectivity_percent = selectivity;
169	600	scenario.projection = projection;
170	600	add(scenario);
171	600	}
172	300	}
173	50	}
174	25	}
175	5	return scenarios;
176	5	}
177
178		inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent,
179	4	Pattern pattern) {
180	4	SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}};
181	4	if (total_rows == 0 \|\| selectivity_percent <= 0) {
182	1	return plan;
183	1	}
184	3	if (selectivity_percent >= 100) {
185	1	plan.selected_rows = total_rows;
186	1	plan.ranges.push_back({.first = 0, .count = total_rows});
187	1	return plan;
188	1	}
189	2	plan.selected_rows = total_rows * static_cast<size_t>(selectivity_percent) / 100;
190	2	if (plan.selected_rows == 0) {
191	0	plan.selected_rows = 1;
192	0	}
193	2	if (pattern == Pattern::CLUSTERED) {
194	1	plan.ranges.push_back({.first = 0, .count = plan.selected_rows});
195	1	return plan;
196	1	}
197
198		// Evenly spaced rows deliberately maximize the number of physical ranges. This is the
199		// adversarial sparse shape that exposes per-run decoder and cursor overhead.
200	101	for (size_t selected = 0; selected < plan.selected_rows; ++selected) {
201	100	const size_t row = selected * total_rows / plan.selected_rows;
202	100	if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) {
203	0	++plan.ranges.back().count;
204	100	} else {
205	100	plan.ranges.push_back({.first = row, .count = 1});
206	100	}
207	100	}
208	1	return plan;
209	2	}
210
211	152	inline std::string to_string(Encoding value) {
212	152	switch (value) {
213	132	case Encoding::PLAIN:
214	132	return "plain";
215	2	case Encoding::DICTIONARY:
216	2	return "dictionary";
217	9	case Encoding::BYTE_STREAM_SPLIT:
218	9	return "byte_stream_split";
219	9	case Encoding::DELTA_BINARY_PACKED:
220	9	return "delta_binary_packed";
221	0	case Encoding::DELTA_LENGTH_BYTE_ARRAY:
222	0	return "delta_length_byte_array";
223	0	case Encoding::DELTA_BYTE_ARRAY:
224	0	return "delta_byte_array";
225	152	}
226	0	return "unknown";
227	152	}
228
229	0	inline std::string to_string(ValueType value) {
230	0	switch (value) {
231	0	case ValueType::INT32:
232	0	return "int32";
233	0	case ValueType::INT64:
234	0	return "int64";
235	0	case ValueType::FLOAT:
236	0	return "float";
237	0	case ValueType::DOUBLE:
238	0	return "double";
239	0	case ValueType::BYTE_ARRAY:
240	0	return "byte_array";
241	0	case ValueType::FIXED_LEN_BYTE_ARRAY:
242	0	return "fixed_len_byte_array";
243	0	}
244	0	return "unknown";
245	0	}
246
247	152	inline std::string to_string(Pattern value) {
248	152	return value == Pattern::CLUSTERED ? "clustered" : "alternating";
249	152	}
250
251	152	inline std::string to_string(Projection value) {
252	152	return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected";
253	152	}
254
255	152	inline std::string to_string(ReaderOperation value) {
256	152	switch (value) {
257	1	case ReaderOperation::OPEN_TO_FIRST_BLOCK:
258	1	return "open_to_first_block";
259	4	case ReaderOperation::FULL_SCAN:
260	4	return "full_scan";
261	144	case ReaderOperation::PREDICATE_SCAN:
262	144	return "predicate_scan";
263	1	case ReaderOperation::COMPLEX_RESIDUAL_SCAN:
264	1	return "complex_residual_scan";
265	1	case ReaderOperation::LIMIT_1:
266	1	return "limit_1";
267	1	case ReaderOperation::LIMIT_1000:
268	1	return "limit_1000";
269	152	}
270	0	return "unknown";
271	152	}
272
273	152	inline std::string reader_scenario_name(const ReaderScenario& scenario) {
274	152	return to_string(scenario.operation) + "/" + to_string(scenario.encoding) + "/null_" +
275	152	std::to_string(scenario.null_percent) + "/" + to_string(scenario.null_pattern) +
276	152	"/sel_" + std::to_string(scenario.selectivity_percent) + "/" +
277	152	to_string(scenario.projection) + "/width_" + std::to_string(scenario.schema_width) +
278	152	"/predicate_" + std::to_string(scenario.predicate_position);
279	152	}
280
281		} // namespace doris::parquet_benchmark