Coverage Report

Created: 2026-08-18 12:46

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/format/parquet/decoder.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "format/parquet/decoder.h"
19
20
#include <cctz/time_zone.h>
21
#include <gen_cpp/parquet_types.h>
22
23
#include "common/cast_set.h"
24
#include "format/parquet/bool_plain_decoder.h"
25
#include "format/parquet/bool_rle_decoder.h"
26
#include "format/parquet/byte_array_dict_decoder.h"
27
#include "format/parquet/byte_array_plain_decoder.h"
28
#include "format/parquet/byte_stream_split_decoder.h"
29
#include "format/parquet/delta_bit_pack_decoder.h"
30
#include "format/parquet/fix_length_dict_decoder.hpp"
31
#include "format/parquet/fix_length_plain_decoder.h"
32
#include "util/rle_encoding.h"
33
34
namespace doris {
35
36
4.34k
BaseDictDecoder::BaseDictDecoder() = default;
37
38
4.34k
BaseDictDecoder::~BaseDictDecoder() = default;
39
40
4.34k
Status BaseDictDecoder::set_data(Slice* data) {
41
4.34k
    _data = data;
42
4.34k
    _offset = 0;
43
4.34k
    uint8_t bit_width = *data->data;
44
4.34k
    _index_batch_decoder = std::make_unique<RleBatchDecoder<uint32_t>>(
45
4.34k
            reinterpret_cast<uint8_t*>(data->data) + 1, static_cast<int>(data->size) - 1,
46
4.34k
            bit_width);
47
4.34k
    return Status::OK();
48
4.34k
}
49
50
1.75k
Status BaseDictDecoder::skip_values(size_t num_values) {
51
1.75k
    _indexes.resize(num_values);
52
1.75k
    _index_batch_decoder->GetBatch(_indexes.data(), cast_set<uint32_t>(num_values));
53
1.75k
    return Status::OK();
54
1.75k
}
55
56
Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding,
57
4.75k
                            std::unique_ptr<Decoder>& decoder) {
58
4.75k
    switch (encoding) {
59
446
    case tparquet::Encoding::PLAIN:
60
446
        switch (type) {
61
277
        case tparquet::Type::BOOLEAN:
62
277
            decoder.reset(new BoolPlainDecoder());
63
277
            break;
64
36
        case tparquet::Type::BYTE_ARRAY:
65
36
            decoder.reset(new ByteArrayPlainDecoder());
66
36
            break;
67
49
        case tparquet::Type::INT32:
68
67
        case tparquet::Type::INT64:
69
67
        case tparquet::Type::INT96:
70
88
        case tparquet::Type::FLOAT:
71
112
        case tparquet::Type::DOUBLE:
72
133
        case tparquet::Type::FIXED_LEN_BYTE_ARRAY:
73
133
            decoder.reset(new FixLengthPlainDecoder());
74
133
            break;
75
0
        default:
76
0
            return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder",
77
0
                                         tparquet::to_string(type), tparquet::to_string(encoding));
78
446
        }
79
446
        break;
80
4.31k
    case tparquet::Encoding::RLE_DICTIONARY:
81
4.31k
        switch (type) {
82
0
        case tparquet::Type::BOOLEAN:
83
0
            return Status::InternalError("Bool type can't has dictionary page");
84
2.09k
        case tparquet::Type::BYTE_ARRAY:
85
2.09k
            decoder.reset(new ByteArrayDictDecoder());
86
2.09k
            break;
87
960
        case tparquet::Type::INT32:
88
960
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT32>());
89
960
            break;
90
325
        case tparquet::Type::INT64:
91
325
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT64>());
92
325
            break;
93
16
        case tparquet::Type::INT96:
94
16
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT96>());
95
16
            break;
96
230
        case tparquet::Type::FLOAT:
97
230
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::FLOAT>());
98
230
            break;
99
235
        case tparquet::Type::DOUBLE:
100
235
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::DOUBLE>());
101
235
            break;
102
454
        case tparquet::Type::FIXED_LEN_BYTE_ARRAY:
103
454
            decoder.reset(new FixLengthDictDecoder<tparquet::Type::FIXED_LEN_BYTE_ARRAY>());
104
454
            break;
105
0
        default:
106
0
            return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder",
107
0
                                         tparquet::to_string(type), tparquet::to_string(encoding));
108
4.31k
        }
109
4.31k
        break;
110
4.31k
    case tparquet::Encoding::RLE:
111
0
        switch (type) {
112
0
        case tparquet::Type::BOOLEAN:
113
0
            decoder.reset(new BoolRLEDecoder());
114
0
            break;
115
0
        default:
116
0
            return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder",
117
0
                                         tparquet::to_string(type), tparquet::to_string(encoding));
118
0
        }
119
0
        break;
120
0
    case tparquet::Encoding::DELTA_BINARY_PACKED:
121
        // Supports only INT32 and INT64.
122
0
        switch (type) {
123
0
        case tparquet::Type::INT32:
124
0
            decoder.reset(new DeltaBitPackDecoder<int32_t>());
125
0
            break;
126
0
        case tparquet::Type::INT64:
127
0
            decoder.reset(new DeltaBitPackDecoder<int64_t>());
128
0
            break;
129
0
        default:
130
0
            return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64");
131
0
        }
132
0
        break;
133
0
    case tparquet::Encoding::DELTA_BYTE_ARRAY:
134
0
        switch (type) {
135
0
        case tparquet::Type::BYTE_ARRAY:
136
0
        case tparquet::Type::FIXED_LEN_BYTE_ARRAY:
137
0
            decoder.reset(new DeltaByteArrayDecoder());
138
0
            break;
139
0
        default:
140
0
            return Status::InternalError(
141
0
                    "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY.");
142
0
        }
143
0
        break;
144
0
    case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY:
145
0
        switch (type) {
146
0
        case tparquet::Type::BYTE_ARRAY:
147
0
            decoder.reset(new DeltaLengthByteArrayDecoder());
148
0
            break;
149
0
        default:
150
0
            return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY.");
151
0
        }
152
0
        break;
153
0
    case tparquet::Encoding::BYTE_STREAM_SPLIT:
154
0
        switch (type) {
155
0
        case tparquet::Type::INT32:
156
0
        case tparquet::Type::INT64:
157
0
        case tparquet::Type::INT96:
158
0
        case tparquet::Type::FLOAT:
159
0
        case tparquet::Type::DOUBLE:
160
0
        case tparquet::Type::FIXED_LEN_BYTE_ARRAY:
161
0
            decoder.reset(new ByteStreamSplitDecoder());
162
0
            break;
163
0
        default:
164
0
            return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder",
165
0
                                         tparquet::to_string(type), tparquet::to_string(encoding));
166
0
        }
167
0
        break;
168
0
    default:
169
0
        return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder",
170
0
                                     tparquet::to_string(encoding), tparquet::to_string(type));
171
4.75k
    }
172
4.75k
    return Status::OK();
173
4.75k
}
174
175
} // namespace doris