Coverage Report

Created: 2026-08-13 14:02

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exprs/function/parse/variant_jsonb_parse.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "exprs/function/parse/variant_jsonb_parse.h"
19
20
#include <array>
21
#include <cstdint>
22
#include <cstring>
23
#include <limits>
24
25
#include "common/exception.h"
26
#include "core/types.h"
27
#include "core/value/variant/variant_parquet_encoding.h"
28
#include "util/jsonb_document.h"
29
#include "util/jsonb_writer.h"
30
31
namespace doris {
32
namespace {
33
34
static_assert(MaxNestingLevel <= VARIANT_MAX_NESTING_DEPTH);
35
36
class BoundedJsonbCursor {
37
public:
38
670
    BoundedJsonbCursor(const char* data, size_t size) : _current(data), _remaining(size) {}
39
40
    template <typename T>
41
1.82k
    T read(const char* description) {
42
1.82k
        require(sizeof(T), description);
43
1.82k
        T value;
44
1.82k
        std::memcpy(&value, _current, sizeof(T));
45
1.82k
        _current += sizeof(T);
46
1.82k
        _remaining -= sizeof(T);
47
1.82k
        return value;
48
1.82k
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIhEET_PKc
Line
Count
Source
41
1.05k
    T read(const char* description) {
42
1.05k
        require(sizeof(T), description);
43
1.05k
        T value;
44
1.05k
        std::memcpy(&value, _current, sizeof(T));
45
1.05k
        _current += sizeof(T);
46
1.05k
        _remaining -= sizeof(T);
47
1.05k
        return value;
48
1.05k
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIaEET_PKc
Line
Count
Source
41
30
    T read(const char* description) {
42
30
        require(sizeof(T), description);
43
30
        T value;
44
30
        std::memcpy(&value, _current, sizeof(T));
45
30
        _current += sizeof(T);
46
30
        _remaining -= sizeof(T);
47
30
        return value;
48
30
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIsEET_PKc
Line
Count
Source
41
5
    T read(const char* description) {
42
5
        require(sizeof(T), description);
43
5
        T value;
44
5
        std::memcpy(&value, _current, sizeof(T));
45
5
        _current += sizeof(T);
46
5
        _remaining -= sizeof(T);
47
5
        return value;
48
5
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIiEET_PKc
Line
Count
Source
41
19
    T read(const char* description) {
42
19
        require(sizeof(T), description);
43
19
        T value;
44
19
        std::memcpy(&value, _current, sizeof(T));
45
19
        _current += sizeof(T);
46
19
        _remaining -= sizeof(T);
47
19
        return value;
48
19
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIlEET_PKc
Line
Count
Source
41
41
    T read(const char* description) {
42
41
        require(sizeof(T), description);
43
41
        T value;
44
41
        std::memcpy(&value, _current, sizeof(T));
45
41
        _current += sizeof(T);
46
41
        _remaining -= sizeof(T);
47
41
        return value;
48
41
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIdEET_PKc
Line
Count
Source
41
10
    T read(const char* description) {
42
10
        require(sizeof(T), description);
43
10
        T value;
44
10
        std::memcpy(&value, _current, sizeof(T));
45
10
        _current += sizeof(T);
46
10
        _remaining -= sizeof(T);
47
10
        return value;
48
10
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIjEET_PKc
Line
Count
Source
41
594
    T read(const char* description) {
42
594
        require(sizeof(T), description);
43
594
        T value;
44
594
        std::memcpy(&value, _current, sizeof(T));
45
594
        _current += sizeof(T);
46
594
        _remaining -= sizeof(T);
47
594
        return value;
48
594
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readItEET_PKc
Line
Count
Source
41
2
    T read(const char* description) {
42
2
        require(sizeof(T), description);
43
2
        T value;
44
2
        std::memcpy(&value, _current, sizeof(T));
45
2
        _current += sizeof(T);
46
2
        _remaining -= sizeof(T);
47
2
        return value;
48
2
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readInEET_PKc
Line
Count
Source
41
66
    T read(const char* description) {
42
66
        require(sizeof(T), description);
43
66
        T value;
44
66
        std::memcpy(&value, _current, sizeof(T));
45
66
        _current += sizeof(T);
46
66
        _remaining -= sizeof(T);
47
66
        return value;
48
66
    }
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIfEET_PKc
Line
Count
Source
41
6
    T read(const char* description) {
42
6
        require(sizeof(T), description);
43
6
        T value;
44
6
        std::memcpy(&value, _current, sizeof(T));
45
6
        _current += sizeof(T);
46
6
        _remaining -= sizeof(T);
47
6
        return value;
48
6
    }
49
50
425
    StringRef read_bytes(size_t size, const char* description) {
51
425
        require(size, description);
52
425
        const StringRef result {_current, size};
53
425
        _current += size;
54
425
        _remaining -= size;
55
425
        return result;
56
425
    }
57
58
372
    BoundedJsonbCursor read_region(size_t size, const char* description) {
59
372
        const StringRef region = read_bytes(size, description);
60
372
        return {region.data, region.size};
61
372
    }
62
63
639
    bool empty() const noexcept { return _remaining == 0; }
64
1
    size_t remaining() const noexcept { return _remaining; }
65
66
private:
67
2.25k
    void require(size_t size, const char* description) const {
68
2.25k
        if (_remaining < size) {
69
169
            throw Exception(ErrorCode::CORRUPTION,
70
169
                            "Truncated JSONB while reading {}: need {} bytes, have {}", description,
71
169
                            size, _remaining);
72
169
        }
73
2.25k
    }
74
75
    const char* _current;
76
    size_t _remaining;
77
};
78
79
template <typename Integer>
80
28
uint32_t decimal_digits(Integer value) {
81
28
    uint32_t digits = 0;
82
213
    do {
83
213
        value /= 10;
84
213
        ++digits;
85
213
    } while (value != 0);
86
28
    return digits;
87
28
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsIiEEjT_
Line
Count
Source
80
5
uint32_t decimal_digits(Integer value) {
81
5
    uint32_t digits = 0;
82
11
    do {
83
11
        value /= 10;
84
11
        ++digits;
85
11
    } while (value != 0);
86
5
    return digits;
87
5
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsIlEEjT_
Line
Count
Source
80
5
uint32_t decimal_digits(Integer value) {
81
5
    uint32_t digits = 0;
82
27
    do {
83
27
        value /= 10;
84
27
        ++digits;
85
27
    } while (value != 0);
86
5
    return digits;
87
5
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsInEEjT_
Line
Count
Source
80
18
uint32_t decimal_digits(Integer value) {
81
18
    uint32_t digits = 0;
82
175
    do {
83
175
        value /= 10;
84
175
        ++digits;
85
175
    } while (value != 0);
86
18
    return digits;
87
18
}
88
89
template <typename Integer>
90
void require_decimal(Integer value, uint32_t precision, uint32_t scale, uint32_t maximum_precision,
91
37
                     uint32_t maximum_scale, const char* description) {
92
37
    if (precision == 0 || precision > maximum_precision) {
93
6
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
6
                        description, precision, maximum_precision);
95
6
    }
96
31
    if (scale > maximum_scale) {
97
3
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
3
                        maximum_scale);
99
3
    }
100
28
    const uint32_t digits = decimal_digits(value);
101
28
    if (digits > precision) {
102
4
        throw Exception(ErrorCode::CORRUPTION,
103
4
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
4
                        digits, precision);
105
4
    }
106
28
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_115require_decimalIiEEvT_jjjjPKc
Line
Count
Source
91
8
                     uint32_t maximum_scale, const char* description) {
92
8
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
6
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
5
    const uint32_t digits = decimal_digits(value);
101
5
    if (digits > precision) {
102
1
        throw Exception(ErrorCode::CORRUPTION,
103
1
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
1
                        digits, precision);
105
1
    }
106
5
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_115require_decimalIlEEvT_jjjjPKc
Line
Count
Source
91
8
                     uint32_t maximum_scale, const char* description) {
92
8
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
6
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
5
    const uint32_t digits = decimal_digits(value);
101
5
    if (digits > precision) {
102
1
        throw Exception(ErrorCode::CORRUPTION,
103
1
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
1
                        digits, precision);
105
1
    }
106
5
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_115require_decimalInEEvT_jjjjPKc
Line
Count
Source
91
21
                     uint32_t maximum_scale, const char* description) {
92
21
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
19
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
18
    const uint32_t digits = decimal_digits(value);
101
18
    if (digits > precision) {
102
2
        throw Exception(ErrorCode::CORRUPTION,
103
2
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
2
                        digits, precision);
105
2
    }
106
18
}
107
108
1.40k
void require_jsonb_depth(uint32_t depth, bool is_container) {
109
1.40k
    if (depth > MaxNestingLevel || (is_container && depth == MaxNestingLevel)) {
110
3
        throw Exception(ErrorCode::INVALID_ARGUMENT,
111
3
                        "JSONB traversal exceeds maximum container nesting {}", MaxNestingLevel);
112
3
    }
113
1.40k
}
114
115
1.08k
void require_variant_depth(uint32_t base_depth, uint32_t relative_depth, bool is_container) {
116
1.08k
    if (base_depth > VARIANT_MAX_NESTING_DEPTH ||
117
1.08k
        relative_depth > VARIANT_MAX_NESTING_DEPTH - base_depth ||
118
1.08k
        (is_container && base_depth + relative_depth == VARIANT_MAX_NESTING_DEPTH)) {
119
0
        throw Exception(ErrorCode::INVALID_ARGUMENT,
120
0
                        "JSONB traversal exceeds maximum Variant container nesting {}",
121
0
                        VARIANT_MAX_NESTING_DEPTH);
122
0
    }
123
1.08k
}
124
125
template <typename Builder>
126
void collect_jsonb_value(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
127
                         uint32_t base_depth);
128
129
template <typename Builder>
130
void collect_jsonb_object(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
131
52
                          uint32_t base_depth) {
132
52
    require_jsonb_depth(relative_depth, true);
133
52
    require_variant_depth(base_depth, relative_depth, true);
134
52
    const auto payload_size = cursor.read<uint32_t>("object payload size");
135
52
    BoundedJsonbCursor payload = cursor.read_region(payload_size, "object payload");
136
52
    auto scope = builder.start_object();
137
82
    while (!payload.empty()) {
138
31
        const auto key_size = payload.read<uint8_t>("object key size");
139
31
        StringRef key;
140
31
        if (key_size == 0) {
141
2
            const auto key_id = payload.read<JsonbKeyValue::keyid_type>("object key id");
142
2
            if (key_id != JsonbKeyValue::sMaxKeyId) {
143
1
                throw Exception(ErrorCode::INVALID_ARGUMENT,
144
1
                                "Cannot resolve external JSONB object key id {} without a "
145
1
                                "dictionary",
146
1
                                key_id);
147
1
            }
148
29
        } else {
149
29
            key = payload.read_bytes(key_size, "object key");
150
29
        }
151
30
        scope.add_key(key);
152
30
        collect_jsonb_value(payload, builder, relative_depth + 1, base_depth);
153
30
    }
154
51
    scope.finish();
155
51
}
156
157
template <typename Builder>
158
void collect_jsonb_array(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
159
326
                         uint32_t base_depth) {
160
326
    require_jsonb_depth(relative_depth, true);
161
326
    require_variant_depth(base_depth, relative_depth, true);
162
326
    const auto payload_size = cursor.read<uint32_t>("array payload size");
163
326
    BoundedJsonbCursor payload = cursor.read_region(payload_size, "array payload");
164
326
    auto scope = builder.start_array();
165
699
    while (!payload.empty()) {
166
373
        collect_jsonb_value(payload, builder, relative_depth + 1, base_depth);
167
373
    }
168
326
    scope.finish();
169
326
}
170
171
template <typename Native, typename Builder>
172
void collect_jsonb_decimal(BoundedJsonbCursor& cursor, Builder& builder, uint8_t width,
173
101
                           uint32_t maximum_precision, const char* description) {
174
101
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
101
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
101
    const auto value = cursor.read<Native>("decimal value");
177
101
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
101
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
101
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalIiNS_19VariantBatchBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
20
                           uint32_t maximum_precision, const char* description) {
174
20
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
20
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
20
    const auto value = cursor.read<Native>("decimal value");
177
20
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
20
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
20
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalIlNS_19VariantBatchBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
24
                           uint32_t maximum_precision, const char* description) {
174
24
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
24
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
24
    const auto value = cursor.read<Native>("decimal value");
177
24
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
24
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
24
}
variant_jsonb_parse.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalInNS_19VariantBatchBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
57
                           uint32_t maximum_precision, const char* description) {
174
57
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
57
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
57
    const auto value = cursor.read<Native>("decimal value");
177
57
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
57
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
57
}
180
181
template <typename Builder>
182
void collect_jsonb_value(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
183
711
                         uint32_t base_depth) {
184
711
    require_jsonb_depth(relative_depth, false);
185
711
    require_variant_depth(base_depth, relative_depth, false);
186
711
    const auto type = static_cast<JsonbType>(cursor.read<uint8_t>("value type"));
187
711
    switch (type) {
188
26
    case JsonbType::T_Null:
189
26
        builder.add_null();
190
26
        return;
191
6
    case JsonbType::T_True:
192
6
        builder.add_bool(true);
193
6
        return;
194
2
    case JsonbType::T_False:
195
2
        builder.add_bool(false);
196
2
        return;
197
30
    case JsonbType::T_Int8:
198
30
        builder.add_int(cursor.read<int8_t>("int8 value"));
199
30
        return;
200
5
    case JsonbType::T_Int16:
201
5
        builder.add_int(cursor.read<int16_t>("int16 value"));
202
5
        return;
203
7
    case JsonbType::T_Int32:
204
7
        builder.add_int(cursor.read<int32_t>("int32 value"));
205
7
        return;
206
25
    case JsonbType::T_Int64:
207
25
        builder.add_int(cursor.read<int64_t>("int64 value"));
208
25
        return;
209
10
    case JsonbType::T_Double:
210
10
        builder.add_double(cursor.read<double>("double value"));
211
10
        return;
212
21
    case JsonbType::T_String: {
213
21
        const auto size = cursor.read<uint32_t>("string size");
214
21
        builder.add_string(cursor.read_bytes(size, "string payload"));
215
21
        return;
216
0
    }
217
11
    case JsonbType::T_Binary: {
218
11
        const auto size = cursor.read<uint32_t>("binary size");
219
11
        builder.add_binary(cursor.read_bytes(size, "binary payload"));
220
11
        return;
221
0
    }
222
52
    case JsonbType::T_Object:
223
52
        collect_jsonb_object(cursor, builder, relative_depth, base_depth);
224
52
        return;
225
326
    case JsonbType::T_Array:
226
326
        collect_jsonb_array(cursor, builder, relative_depth, base_depth);
227
326
        return;
228
25
    case JsonbType::T_Int128:
229
25
        builder.add_largeint(cursor.read<__int128>("int128 value"));
230
25
        return;
231
6
    case JsonbType::T_Float:
232
6
        builder.add_float(cursor.read<float>("float value"));
233
6
        return;
234
20
    case JsonbType::T_Decimal32:
235
20
        collect_jsonb_decimal<int32_t>(cursor, builder, 4, 9, "Decimal32");
236
20
        return;
237
24
    case JsonbType::T_Decimal64:
238
24
        collect_jsonb_decimal<int64_t>(cursor, builder, 8, 18, "Decimal64");
239
24
        return;
240
57
    case JsonbType::T_Decimal128:
241
57
        collect_jsonb_decimal<__int128>(cursor, builder, 16, 38, "Decimal128");
242
57
        return;
243
41
    case JsonbType::T_Decimal256:
244
41
        throw Exception(ErrorCode::INVALID_ARGUMENT,
245
41
                        "Conversion from JSONB DECIMAL256 to Variant V2 is not supported");
246
0
    case JsonbType::NUM_TYPES:
247
0
        break;
248
711
    }
249
2
    throw Exception(ErrorCode::CORRUPTION, "Unknown JSONB value type {}",
250
2
                    static_cast<uint8_t>(type));
251
711
}
252
253
template <typename Builder>
254
void collect_jsonb_document(StringRef document, Builder& builder, uint32_t initial_depth = 0,
255
314
                            bool* root_is_null = nullptr) {
256
314
    if (root_is_null != nullptr) {
257
0
        *root_is_null = false;
258
0
    }
259
314
    if (document.data == nullptr && document.size != 0) {
260
1
        throw Exception(ErrorCode::INVALID_ARGUMENT,
261
1
                        "JSONB input has a null data pointer for {} bytes", document.size);
262
1
    }
263
313
    if (document.size == 0) {
264
2
        if (root_is_null != nullptr) {
265
0
            *root_is_null = true;
266
0
        }
267
2
        builder.add_null();
268
2
        return;
269
2
    }
270
271
311
    BoundedJsonbCursor cursor(document.data, document.size);
272
311
    const auto version = cursor.read<uint8_t>("document version");
273
311
    if (version != JSONB_VER) {
274
2
        throw Exception(ErrorCode::CORRUPTION, "Unsupported JSONB version {}, expected {}", version,
275
2
                        JSONB_VER);
276
2
    }
277
309
    if (root_is_null != nullptr && document.size >= sizeof(uint8_t) * 2) {
278
0
        *root_is_null =
279
0
                static_cast<JsonbType>(static_cast<uint8_t>(document.data[1])) == JsonbType::T_Null;
280
0
    }
281
309
    collect_jsonb_value(cursor, builder, 0, initial_depth);
282
309
    if (!cursor.empty()) {
283
1
        throw Exception(ErrorCode::CORRUPTION,
284
1
                        "JSONB document has {} trailing bytes after its root value",
285
1
                        cursor.remaining());
286
1
    }
287
309
}
288
289
487
void require_jsonb_write(bool written, const char* operation) {
290
487
    if (!written) {
291
0
        throw Exception(ErrorCode::INVALID_ARGUMENT, "JSONB writer rejected {}", operation);
292
0
    }
293
487
}
294
295
18
void write_jsonb_string(JsonbWriter& writer, StringRef value, const char* description) {
296
18
    variant_json::require_valid_json_utf8(value, description);
297
18
    require_jsonb_write(writer.writeStartString(), "string start");
298
18
    require_jsonb_write(writer.writeString(value.data, value.size), "string payload");
299
18
    require_jsonb_write(writer.writeEndString(), "string end");
300
18
}
301
302
11
void write_jsonb_string(JsonbWriter& writer, const variant_json::FormattedScalar& value) {
303
11
    write_jsonb_string(writer, {value.bytes.data(), value.size}, "formatted string");
304
11
}
305
306
2
void write_jsonb_binary(JsonbWriter& writer, StringRef value) {
307
2
    require_jsonb_write(writer.writeStartBinary(), "binary start");
308
2
    require_jsonb_write(writer.writeBinary(value.data, value.size), "binary payload");
309
2
    require_jsonb_write(writer.writeEndBinary(), "binary end");
310
2
}
311
312
class VariantToJsonbConverter {
313
public:
314
    VariantToJsonbConverter(JsonbWriter& writer, const VariantJsonFormatOptions& options)
315
47
            : _writer(writer), _options(options) {}
316
317
316
    void write(VariantRef value, uint32_t depth) {
318
316
        const VariantBasicType type = value.basic_type();
319
316
        const bool is_container =
320
316
                type == VariantBasicType::OBJECT || type == VariantBasicType::ARRAY;
321
316
        require_jsonb_depth(depth, is_container);
322
316
        switch (type) {
323
5
        case VariantBasicType::SHORT_STRING:
324
5
            write_jsonb_string(_writer, value.get_string(), "Variant string");
325
5
            return;
326
84
        case VariantBasicType::PRIMITIVE:
327
84
            write_primitive(value);
328
84
            return;
329
16
        case VariantBasicType::OBJECT:
330
16
            write_object(value, depth);
331
16
            return;
332
210
        case VariantBasicType::ARRAY:
333
210
            write_array(value, depth);
334
210
            return;
335
316
        }
336
316
    }
337
338
private:
339
12
    void write_decimal(VariantDecimal decimal) {
340
12
        uint32_t precision = 38;
341
12
        if (decimal.width == 4) {
342
4
            precision = 9;
343
8
        } else if (decimal.width == 8) {
344
3
            precision = 18;
345
3
        }
346
12
        require_decimal(decimal.unscaled, precision, decimal.scale, precision, 38,
347
12
                        "Variant decimal");
348
12
        if (decimal.width == 4) {
349
3
            require_jsonb_write(
350
3
                    _writer.writeDecimal(Decimal32 {static_cast<int32_t>(decimal.unscaled)},
351
3
                                         precision, decimal.scale),
352
3
                    "Decimal32 value");
353
9
        } else if (decimal.width == 8) {
354
3
            require_jsonb_write(
355
3
                    _writer.writeDecimal(Decimal64 {static_cast<int64_t>(decimal.unscaled)},
356
3
                                         precision, decimal.scale),
357
3
                    "Decimal64 value");
358
6
        } else {
359
6
            require_jsonb_write(
360
6
                    _writer.writeDecimal(Decimal128V3 {decimal.unscaled}, precision, decimal.scale),
361
6
                    "Decimal128 value");
362
6
        }
363
12
    }
364
365
84
    void write_primitive(VariantRef value) {
366
84
        switch (value.primitive_id()) {
367
11
        case VariantPrimitiveId::NULL_VALUE:
368
11
            require_jsonb_write(_writer.writeNull(), "null value");
369
11
            return;
370
4
        case VariantPrimitiveId::TRUE_VALUE:
371
4
            require_jsonb_write(_writer.writeBool(true), "true value");
372
4
            return;
373
3
        case VariantPrimitiveId::FALSE_VALUE:
374
3
            require_jsonb_write(_writer.writeBool(false), "false value");
375
3
            return;
376
28
        case VariantPrimitiveId::INT8:
377
28
            require_jsonb_write(_writer.writeInt8(static_cast<int8_t>(value.get_int())),
378
28
                                "int8 value");
379
28
            return;
380
2
        case VariantPrimitiveId::INT16:
381
2
            require_jsonb_write(_writer.writeInt16(static_cast<int16_t>(value.get_int())),
382
2
                                "int16 value");
383
2
            return;
384
2
        case VariantPrimitiveId::INT32:
385
2
            require_jsonb_write(_writer.writeInt32(static_cast<int32_t>(value.get_int())),
386
2
                                "int32 value");
387
2
            return;
388
2
        case VariantPrimitiveId::INT64:
389
2
            require_jsonb_write(_writer.writeInt64(value.get_int()), "int64 value");
390
2
            return;
391
2
        case VariantPrimitiveId::DOUBLE:
392
2
            require_jsonb_write(_writer.writeDouble(value.get_double()), "double value");
393
2
            return;
394
4
        case VariantPrimitiveId::DECIMAL4:
395
7
        case VariantPrimitiveId::DECIMAL8:
396
12
        case VariantPrimitiveId::DECIMAL16:
397
12
            write_decimal(value.get_decimal());
398
12
            return;
399
1
        case VariantPrimitiveId::DATE:
400
1
            write_jsonb_string(_writer, variant_json::format_json_date(value.get_date()));
401
1
            return;
402
2
        case VariantPrimitiveId::TIMESTAMP_MICROS:
403
2
            write_jsonb_string(_writer,
404
2
                               variant_json::format_json_timestamp(value.get_timestamp_micros(), 6,
405
2
                                                                   true, _options.timezone));
406
2
            return;
407
2
        case VariantPrimitiveId::TIMESTAMP_NTZ_MICROS:
408
2
            write_jsonb_string(_writer,
409
2
                               variant_json::format_json_timestamp(value.get_timestamp_ntz_micros(),
410
2
                                                                   6, false, nullptr));
411
2
            return;
412
2
        case VariantPrimitiveId::FLOAT:
413
2
            require_jsonb_write(_writer.writeFloat(value.get_float()), "float value");
414
2
            return;
415
2
        case VariantPrimitiveId::BINARY:
416
2
            write_jsonb_binary(_writer, value.get_binary());
417
2
            return;
418
2
        case VariantPrimitiveId::STRING:
419
2
            write_jsonb_string(_writer, value.get_string(), "Variant string");
420
2
            return;
421
2
        case VariantPrimitiveId::TIME_NTZ_MICROS:
422
2
            write_jsonb_string(_writer,
423
2
                               variant_json::format_json_time_micros(value.get_time_ntz_micros()));
424
2
            return;
425
2
        case VariantPrimitiveId::TIMESTAMP_NANOS:
426
2
            write_jsonb_string(_writer,
427
2
                               variant_json::format_json_timestamp(value.get_timestamp_nanos(), 9,
428
2
                                                                   true, _options.timezone));
429
2
            return;
430
2
        case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS:
431
2
            write_jsonb_string(
432
2
                    _writer, variant_json::format_json_timestamp(value.get_timestamp_ntz_nanos(), 9,
433
2
                                                                 false, nullptr));
434
2
            return;
435
1
        case VariantPrimitiveId::UUID:
436
1
            write_jsonb_string(_writer, variant_json::format_json_uuid(value.get_uuid()));
437
1
            return;
438
84
        }
439
0
        throw Exception(ErrorCode::CORRUPTION, "Unknown Variant primitive id");
440
84
    }
441
442
16
    void write_object(VariantRef value, uint32_t depth) {
443
16
        require_jsonb_write(_writer.writeStartObject(), "object start");
444
16
        const uint32_t count = value.num_elements();
445
16
        StringRef previous_key;
446
32
        for (uint32_t index = 0; index < count; ++index) {
447
17
            uint32_t field_id = 0;
448
17
            const VariantRef child = value.object_value_at(index, &field_id);
449
17
            const StringRef key = value.metadata.key_at(field_id);
450
17
            variant_json::require_json_object_key(key, previous_key, index);
451
17
            if (key.size > std::numeric_limits<uint8_t>::max()) {
452
1
                throw Exception(ErrorCode::INVALID_ARGUMENT,
453
1
                                "Variant object key length {} exceeds JSONB maximum {}", key.size,
454
1
                                std::numeric_limits<uint8_t>::max());
455
1
            }
456
16
            require_jsonb_write(_writer.writeKey(key.data, static_cast<uint8_t>(key.size)),
457
16
                                "object key");
458
16
            write(child, depth + 1);
459
16
            previous_key = key;
460
16
        }
461
15
        require_jsonb_write(_writer.writeEndObject(), "object end");
462
15
    }
463
464
210
    void write_array(VariantRef value, uint32_t depth) {
465
210
        require_jsonb_write(_writer.writeStartArray(), "array start");
466
210
        const uint32_t count = value.num_elements();
467
465
        for (uint32_t index = 0; index < count; ++index) {
468
255
            write(value.array_at(index), depth + 1);
469
255
        }
470
210
        require_jsonb_write(_writer.writeEndArray(), "array end");
471
210
    }
472
473
    JsonbWriter& _writer;
474
    const VariantJsonFormatOptions& _options;
475
};
476
477
} // namespace
478
479
struct JsonbToVariantEncoder::Impl {
480
    enum class State : uint8_t { COLLECTING, FINISHED, FAILED };
481
482
15
    explicit Impl(VariantBatchBuilder::ReserveHint hint) : builder(hint) {}
483
484
33
    void require_collecting() const {
485
33
        if (state == State::FINISHED) {
486
2
            throw Exception(ErrorCode::INVALID_ARGUMENT,
487
2
                            "Variant JSONB encoder is already finished");
488
2
        }
489
31
        if (state == State::FAILED) {
490
3
            throw Exception(ErrorCode::INVALID_ARGUMENT,
491
3
                            "Variant JSONB encoder is in a terminal failed state");
492
3
        }
493
31
    }
494
495
    VariantBatchBuilder builder;
496
    State state = State::COLLECTING;
497
};
498
499
JsonbToVariantEncoder::JsonbToVariantEncoder()
500
11
        : JsonbToVariantEncoder(VariantBatchBuilder::ReserveHint {}) {}
501
JsonbToVariantEncoder::JsonbToVariantEncoder(VariantBatchBuilder::ReserveHint hint)
502
15
        : _impl(std::make_unique<Impl>(hint)) {}
503
15
JsonbToVariantEncoder::~JsonbToVariantEncoder() = default;
504
0
JsonbToVariantEncoder::JsonbToVariantEncoder(JsonbToVariantEncoder&&) noexcept = default;
505
0
JsonbToVariantEncoder& JsonbToVariantEncoder::operator=(JsonbToVariantEncoder&&) noexcept = default;
506
507
1
void JsonbToVariantEncoder::add_null() {
508
1
    _impl->require_collecting();
509
1
    try {
510
1
        auto row = _impl->builder.begin_row();
511
1
        row.add_null();
512
1
        row.finish();
513
1
    } catch (...) {
514
0
        _impl->state = Impl::State::FAILED;
515
0
        throw;
516
0
    }
517
1
}
518
519
18
void JsonbToVariantEncoder::add_jsonb(StringRef document) {
520
18
    _impl->require_collecting();
521
18
    try {
522
18
        auto row = _impl->builder.begin_row();
523
18
        collect_jsonb_document(document, row);
524
18
        row.finish();
525
18
    } catch (...) {
526
4
        _impl->state = Impl::State::FAILED;
527
4
        throw;
528
4
    }
529
18
}
530
531
14
VariantBatchBuilder JsonbToVariantEncoder::finish_batch() {
532
14
    _impl->require_collecting();
533
14
    try {
534
14
        VariantBatchBuilder block = _impl->builder.finish_batch();
535
14
        _impl->state = Impl::State::FINISHED;
536
14
        return block;
537
14
    } catch (...) {
538
0
        _impl->state = Impl::State::FAILED;
539
0
        throw;
540
0
    }
541
14
}
542
543
void jsonb_to_variant(StringRef document, VariantBatchBuilder::Row& row, uint32_t initial_depth,
544
298
                      bool* root_is_null) {
545
298
    try {
546
298
        collect_jsonb_document(document, row, initial_depth, root_is_null);
547
298
    } catch (...) {
548
231
        row.abort();
549
231
        throw;
550
231
    }
551
298
}
552
553
void variant_to_jsonb(VariantRef value, JsonbWriter& writer,
554
50
                      const VariantJsonFormatOptions& options) {
555
50
    writer.reset();
556
50
    try {
557
50
        if (value.metadata.version() != VARIANT_ENCODING_VERSION) {
558
0
            throw Exception(ErrorCode::CORRUPTION, "Unsupported Variant metadata version {}",
559
0
                            value.metadata.version());
560
0
        }
561
50
        static_cast<void>(value.metadata.dict_size());
562
50
        variant_json::require_exact_json_value(value);
563
50
        VariantToJsonbConverter(writer, options).write(value, 0);
564
50
    } catch (...) {
565
10
        writer.reset();
566
10
        throw;
567
10
    }
568
50
}
569
570
} // namespace doris