be/src/format/parquet/decoder.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "format/parquet/decoder.h" |
19 | | |
20 | | #include <cctz/time_zone.h> |
21 | | #include <gen_cpp/parquet_types.h> |
22 | | |
23 | | #include "common/cast_set.h" |
24 | | #include "format/parquet/bool_plain_decoder.h" |
25 | | #include "format/parquet/bool_rle_decoder.h" |
26 | | #include "format/parquet/byte_array_dict_decoder.h" |
27 | | #include "format/parquet/byte_array_plain_decoder.h" |
28 | | #include "format/parquet/byte_stream_split_decoder.h" |
29 | | #include "format/parquet/delta_bit_pack_decoder.h" |
30 | | #include "format/parquet/fix_length_dict_decoder.hpp" |
31 | | #include "format/parquet/fix_length_plain_decoder.h" |
32 | | #include "util/rle_encoding.h" |
33 | | |
34 | | namespace doris { |
35 | | |
36 | 125 | BaseDictDecoder::BaseDictDecoder() = default; |
37 | | |
38 | 125 | BaseDictDecoder::~BaseDictDecoder() = default; |
39 | | |
40 | 101 | Status BaseDictDecoder::set_data(Slice* data) { |
41 | 101 | _data = data; |
42 | 101 | _offset = 0; |
43 | 101 | uint8_t bit_width = *data->data; |
44 | 101 | _index_batch_decoder = std::make_unique<RleBatchDecoder<uint32_t>>( |
45 | 101 | reinterpret_cast<uint8_t*>(data->data) + 1, static_cast<int>(data->size) - 1, |
46 | 101 | bit_width); |
47 | 101 | return Status::OK(); |
48 | 101 | } |
49 | | |
50 | 2 | Status BaseDictDecoder::skip_values(size_t num_values) { |
51 | 2 | _indexes.resize(num_values); |
52 | 2 | _index_batch_decoder->GetBatch(_indexes.data(), cast_set<uint32_t>(num_values)); |
53 | 2 | return Status::OK(); |
54 | 2 | } |
55 | | |
56 | | Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, |
57 | 225 | std::unique_ptr<Decoder>& decoder) { |
58 | 225 | switch (encoding) { |
59 | 138 | case tparquet::Encoding::PLAIN: |
60 | 138 | switch (type) { |
61 | 31 | case tparquet::Type::BOOLEAN: |
62 | 31 | decoder.reset(new BoolPlainDecoder()); |
63 | 31 | break; |
64 | 24 | case tparquet::Type::BYTE_ARRAY: |
65 | 24 | decoder.reset(new ByteArrayPlainDecoder()); |
66 | 24 | break; |
67 | 37 | case tparquet::Type::INT32: |
68 | 47 | case tparquet::Type::INT64: |
69 | 47 | case tparquet::Type::INT96: |
70 | 64 | case tparquet::Type::FLOAT: |
71 | 82 | case tparquet::Type::DOUBLE: |
72 | 83 | case tparquet::Type::FIXED_LEN_BYTE_ARRAY: |
73 | 83 | decoder.reset(new FixLengthPlainDecoder()); |
74 | 83 | break; |
75 | 0 | default: |
76 | 0 | return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", |
77 | 0 | tparquet::to_string(type), tparquet::to_string(encoding)); |
78 | 138 | } |
79 | 138 | break; |
80 | 138 | case tparquet::Encoding::RLE_DICTIONARY: |
81 | 87 | switch (type) { |
82 | 0 | case tparquet::Type::BOOLEAN: |
83 | 0 | return Status::InternalError("Bool type can't has dictionary page"); |
84 | 23 | case tparquet::Type::BYTE_ARRAY: |
85 | 23 | decoder.reset(new ByteArrayDictDecoder()); |
86 | 23 | break; |
87 | 55 | case tparquet::Type::INT32: |
88 | 55 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT32>()); |
89 | 55 | break; |
90 | 7 | case tparquet::Type::INT64: |
91 | 7 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT64>()); |
92 | 7 | break; |
93 | 0 | case tparquet::Type::INT96: |
94 | 0 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::INT96>()); |
95 | 0 | break; |
96 | 1 | case tparquet::Type::FLOAT: |
97 | 1 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::FLOAT>()); |
98 | 1 | break; |
99 | 1 | case tparquet::Type::DOUBLE: |
100 | 1 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::DOUBLE>()); |
101 | 1 | break; |
102 | 0 | case tparquet::Type::FIXED_LEN_BYTE_ARRAY: |
103 | 0 | decoder.reset(new FixLengthDictDecoder<tparquet::Type::FIXED_LEN_BYTE_ARRAY>()); |
104 | 0 | break; |
105 | 0 | default: |
106 | 0 | return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", |
107 | 0 | tparquet::to_string(type), tparquet::to_string(encoding)); |
108 | 87 | } |
109 | 87 | break; |
110 | 87 | case tparquet::Encoding::RLE: |
111 | 0 | switch (type) { |
112 | 0 | case tparquet::Type::BOOLEAN: |
113 | 0 | decoder.reset(new BoolRLEDecoder()); |
114 | 0 | break; |
115 | 0 | default: |
116 | 0 | return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", |
117 | 0 | tparquet::to_string(type), tparquet::to_string(encoding)); |
118 | 0 | } |
119 | 0 | break; |
120 | 0 | case tparquet::Encoding::DELTA_BINARY_PACKED: |
121 | | // Supports only INT32 and INT64. |
122 | 0 | switch (type) { |
123 | 0 | case tparquet::Type::INT32: |
124 | 0 | decoder.reset(new DeltaBitPackDecoder<int32_t>()); |
125 | 0 | break; |
126 | 0 | case tparquet::Type::INT64: |
127 | 0 | decoder.reset(new DeltaBitPackDecoder<int64_t>()); |
128 | 0 | break; |
129 | 0 | default: |
130 | 0 | return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); |
131 | 0 | } |
132 | 0 | break; |
133 | 0 | case tparquet::Encoding::DELTA_BYTE_ARRAY: |
134 | 0 | switch (type) { |
135 | 0 | case tparquet::Type::BYTE_ARRAY: |
136 | 0 | case tparquet::Type::FIXED_LEN_BYTE_ARRAY: |
137 | 0 | decoder.reset(new DeltaByteArrayDecoder()); |
138 | 0 | break; |
139 | 0 | default: |
140 | 0 | return Status::InternalError( |
141 | 0 | "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); |
142 | 0 | } |
143 | 0 | break; |
144 | 0 | case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: |
145 | 0 | switch (type) { |
146 | 0 | case tparquet::Type::BYTE_ARRAY: |
147 | 0 | decoder.reset(new DeltaLengthByteArrayDecoder()); |
148 | 0 | break; |
149 | 0 | default: |
150 | 0 | return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); |
151 | 0 | } |
152 | 0 | break; |
153 | 0 | case tparquet::Encoding::BYTE_STREAM_SPLIT: |
154 | 0 | switch (type) { |
155 | 0 | case tparquet::Type::INT32: |
156 | 0 | case tparquet::Type::INT64: |
157 | 0 | case tparquet::Type::INT96: |
158 | 0 | case tparquet::Type::FLOAT: |
159 | 0 | case tparquet::Type::DOUBLE: |
160 | 0 | case tparquet::Type::FIXED_LEN_BYTE_ARRAY: |
161 | 0 | decoder.reset(new ByteStreamSplitDecoder()); |
162 | 0 | break; |
163 | 0 | default: |
164 | 0 | return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", |
165 | 0 | tparquet::to_string(type), tparquet::to_string(encoding)); |
166 | 0 | } |
167 | 0 | break; |
168 | 0 | default: |
169 | 0 | return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", |
170 | 0 | tparquet::to_string(encoding), tparquet::to_string(type)); |
171 | 225 | } |
172 | 225 | return Status::OK(); |
173 | 225 | } |
174 | | |
175 | | } // namespace doris |