Coverage Report

Created: 2026-07-17 03:38

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/variant/variant_jsonb.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/variant/variant_jsonb.h"
19
20
#include <array>
21
#include <cstdint>
22
#include <cstring>
23
#include <limits>
24
25
#include "common/exception.h"
26
#include "core/types.h"
27
#include "util/jsonb_document.h"
28
#include "util/jsonb_writer.h"
29
#include "util/variant/variant_encoding.h"
30
31
namespace doris {
32
namespace {
33
34
static_assert(MaxNestingLevel <= VARIANT_MAX_NESTING_DEPTH);
35
36
class BoundedJsonbCursor {
37
public:
38
585
    BoundedJsonbCursor(const char* data, size_t size) : _current(data), _remaining(size) {}
39
40
    template <typename T>
41
1.60k
    T read(const char* description) {
42
1.60k
        require(sizeof(T), description);
43
1.60k
        T value;
44
1.60k
        std::memcpy(&value, _current, sizeof(T));
45
1.60k
        _current += sizeof(T);
46
1.60k
        _remaining -= sizeof(T);
47
1.60k
        return value;
48
1.60k
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIhEET_PKc
Line
Count
Source
41
905
    T read(const char* description) {
42
905
        require(sizeof(T), description);
43
905
        T value;
44
905
        std::memcpy(&value, _current, sizeof(T));
45
905
        _current += sizeof(T);
46
905
        _remaining -= sizeof(T);
47
905
        return value;
48
905
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIaEET_PKc
Line
Count
Source
41
8
    T read(const char* description) {
42
8
        require(sizeof(T), description);
43
8
        T value;
44
8
        std::memcpy(&value, _current, sizeof(T));
45
8
        _current += sizeof(T);
46
8
        _remaining -= sizeof(T);
47
8
        return value;
48
8
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIsEET_PKc
Line
Count
Source
41
5
    T read(const char* description) {
42
5
        require(sizeof(T), description);
43
5
        T value;
44
5
        std::memcpy(&value, _current, sizeof(T));
45
5
        _current += sizeof(T);
46
5
        _remaining -= sizeof(T);
47
5
        return value;
48
5
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIiEET_PKc
Line
Count
Source
41
19
    T read(const char* description) {
42
19
        require(sizeof(T), description);
43
19
        T value;
44
19
        std::memcpy(&value, _current, sizeof(T));
45
19
        _current += sizeof(T);
46
19
        _remaining -= sizeof(T);
47
19
        return value;
48
19
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIlEET_PKc
Line
Count
Source
41
28
    T read(const char* description) {
42
28
        require(sizeof(T), description);
43
28
        T value;
44
28
        std::memcpy(&value, _current, sizeof(T));
45
28
        _current += sizeof(T);
46
28
        _remaining -= sizeof(T);
47
28
        return value;
48
28
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIdEET_PKc
Line
Count
Source
41
10
    T read(const char* description) {
42
10
        require(sizeof(T), description);
43
10
        T value;
44
10
        std::memcpy(&value, _current, sizeof(T));
45
10
        _current += sizeof(T);
46
10
        _remaining -= sizeof(T);
47
10
        return value;
48
10
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIjEET_PKc
Line
Count
Source
41
559
    T read(const char* description) {
42
559
        require(sizeof(T), description);
43
559
        T value;
44
559
        std::memcpy(&value, _current, sizeof(T));
45
559
        _current += sizeof(T);
46
559
        _remaining -= sizeof(T);
47
559
        return value;
48
559
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readItEET_PKc
Line
Count
Source
41
2
    T read(const char* description) {
42
2
        require(sizeof(T), description);
43
2
        T value;
44
2
        std::memcpy(&value, _current, sizeof(T));
45
2
        _current += sizeof(T);
46
2
        _remaining -= sizeof(T);
47
2
        return value;
48
2
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readInEET_PKc
Line
Count
Source
41
66
    T read(const char* description) {
42
66
        require(sizeof(T), description);
43
66
        T value;
44
66
        std::memcpy(&value, _current, sizeof(T));
45
66
        _current += sizeof(T);
46
66
        _remaining -= sizeof(T);
47
66
        return value;
48
66
    }
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_118BoundedJsonbCursor4readIfEET_PKc
Line
Count
Source
41
6
    T read(const char* description) {
42
6
        require(sizeof(T), description);
43
6
        T value;
44
6
        std::memcpy(&value, _current, sizeof(T));
45
6
        _current += sizeof(T);
46
6
        _remaining -= sizeof(T);
47
6
        return value;
48
6
    }
49
50
378
    StringRef read_bytes(size_t size, const char* description) {
51
378
        require(size, description);
52
378
        const StringRef result {_current, size};
53
378
        _current += size;
54
378
        _remaining -= size;
55
378
        return result;
56
378
    }
57
58
341
    BoundedJsonbCursor read_region(size_t size, const char* description) {
59
341
        const StringRef region = read_bytes(size, description);
60
341
        return {region.data, region.size};
61
341
    }
62
63
526
    bool empty() const noexcept { return _remaining == 0; }
64
1
    size_t remaining() const noexcept { return _remaining; }
65
66
private:
67
1.98k
    void require(size_t size, const char* description) const {
68
1.98k
        if (_remaining < size) {
69
169
            throw Exception(ErrorCode::CORRUPTION,
70
169
                            "Truncated JSONB while reading {}: need {} bytes, have {}", description,
71
169
                            size, _remaining);
72
169
        }
73
1.98k
    }
74
75
    const char* _current;
76
    size_t _remaining;
77
};
78
79
template <typename Integer>
80
28
uint32_t decimal_digits(Integer value) {
81
28
    uint32_t digits = 0;
82
213
    do {
83
213
        value /= 10;
84
213
        ++digits;
85
213
    } while (value != 0);
86
28
    return digits;
87
28
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsInEEjT_
Line
Count
Source
80
18
uint32_t decimal_digits(Integer value) {
81
18
    uint32_t digits = 0;
82
175
    do {
83
175
        value /= 10;
84
175
        ++digits;
85
175
    } while (value != 0);
86
18
    return digits;
87
18
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsIiEEjT_
Line
Count
Source
80
5
uint32_t decimal_digits(Integer value) {
81
5
    uint32_t digits = 0;
82
11
    do {
83
11
        value /= 10;
84
11
        ++digits;
85
11
    } while (value != 0);
86
5
    return digits;
87
5
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_114decimal_digitsIlEEjT_
Line
Count
Source
80
5
uint32_t decimal_digits(Integer value) {
81
5
    uint32_t digits = 0;
82
27
    do {
83
27
        value /= 10;
84
27
        ++digits;
85
27
    } while (value != 0);
86
5
    return digits;
87
5
}
88
89
template <typename Integer>
90
void require_decimal(Integer value, uint32_t precision, uint32_t scale, uint32_t maximum_precision,
91
37
                     uint32_t maximum_scale, const char* description) {
92
37
    if (precision == 0 || precision > maximum_precision) {
93
6
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
6
                        description, precision, maximum_precision);
95
6
    }
96
31
    if (scale > maximum_scale) {
97
3
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
3
                        maximum_scale);
99
3
    }
100
28
    const uint32_t digits = decimal_digits(value);
101
28
    if (digits > precision) {
102
4
        throw Exception(ErrorCode::CORRUPTION,
103
4
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
4
                        digits, precision);
105
4
    }
106
28
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_115require_decimalInEEvT_jjjjPKc
Line
Count
Source
91
21
                     uint32_t maximum_scale, const char* description) {
92
21
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
19
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
18
    const uint32_t digits = decimal_digits(value);
101
18
    if (digits > precision) {
102
2
        throw Exception(ErrorCode::CORRUPTION,
103
2
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
2
                        digits, precision);
105
2
    }
106
18
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_115require_decimalIiEEvT_jjjjPKc
Line
Count
Source
91
8
                     uint32_t maximum_scale, const char* description) {
92
8
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
6
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
5
    const uint32_t digits = decimal_digits(value);
101
5
    if (digits > precision) {
102
1
        throw Exception(ErrorCode::CORRUPTION,
103
1
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
1
                        digits, precision);
105
1
    }
106
5
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_115require_decimalIlEEvT_jjjjPKc
Line
Count
Source
91
8
                     uint32_t maximum_scale, const char* description) {
92
8
    if (precision == 0 || precision > maximum_precision) {
93
2
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} precision {} is outside [1, {}]",
94
2
                        description, precision, maximum_precision);
95
2
    }
96
6
    if (scale > maximum_scale) {
97
1
        throw Exception(ErrorCode::CORRUPTION, "JSONB {} scale {} exceeds {}", description, scale,
98
1
                        maximum_scale);
99
1
    }
100
5
    const uint32_t digits = decimal_digits(value);
101
5
    if (digits > precision) {
102
1
        throw Exception(ErrorCode::CORRUPTION,
103
1
                        "JSONB {} value has {} digits but declared precision is {}", description,
104
1
                        digits, precision);
105
1
    }
106
5
}
107
108
1.25k
void require_jsonb_depth(uint32_t depth, bool is_container) {
109
1.25k
    if (depth > MaxNestingLevel || (is_container && depth == MaxNestingLevel)) {
110
3
        throw Exception(ErrorCode::INVALID_ARGUMENT,
111
3
                        "JSONB traversal exceeds maximum container nesting {}", MaxNestingLevel);
112
3
    }
113
1.25k
}
114
115
974
void require_variant_depth(uint32_t base_depth, uint32_t relative_depth, bool is_container) {
116
974
    if (base_depth > VARIANT_MAX_NESTING_DEPTH ||
117
974
        relative_depth > VARIANT_MAX_NESTING_DEPTH - base_depth ||
118
974
        (is_container && base_depth + relative_depth == VARIANT_MAX_NESTING_DEPTH)) {
119
0
        throw Exception(ErrorCode::INVALID_ARGUMENT,
120
0
                        "JSONB traversal exceeds maximum Variant container nesting {}",
121
0
                        VARIANT_MAX_NESTING_DEPTH);
122
0
    }
123
974
}
124
125
template <typename Builder>
126
void collect_jsonb_value(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
127
                         uint32_t base_depth);
128
129
template <typename Builder>
130
void collect_jsonb_object(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
131
33
                          uint32_t base_depth) {
132
33
    require_jsonb_depth(relative_depth, true);
133
33
    require_variant_depth(base_depth, relative_depth, true);
134
33
    const auto payload_size = cursor.read<uint32_t>("object payload size");
135
33
    BoundedJsonbCursor payload = cursor.read_region(payload_size, "object payload");
136
33
    auto scope = builder.start_object();
137
51
    while (!payload.empty()) {
138
19
        const auto key_size = payload.read<uint8_t>("object key size");
139
19
        StringRef key;
140
19
        if (key_size == 0) {
141
2
            const auto key_id = payload.read<JsonbKeyValue::keyid_type>("object key id");
142
2
            if (key_id != JsonbKeyValue::sMaxKeyId) {
143
1
                throw Exception(ErrorCode::INVALID_ARGUMENT,
144
1
                                "Cannot resolve external JSONB object key id {} without a "
145
1
                                "dictionary",
146
1
                                key_id);
147
1
            }
148
17
        } else {
149
17
            key = payload.read_bytes(key_size, "object key");
150
17
        }
151
18
        scope.add_key(key);
152
18
        collect_jsonb_value(payload, builder, relative_depth + 1, base_depth);
153
18
    }
154
32
    scope.finish();
155
32
}
156
157
template <typename Builder>
158
void collect_jsonb_array(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
159
314
                         uint32_t base_depth) {
160
314
    require_jsonb_depth(relative_depth, true);
161
314
    require_variant_depth(base_depth, relative_depth, true);
162
314
    const auto payload_size = cursor.read<uint32_t>("array payload size");
163
314
    BoundedJsonbCursor payload = cursor.read_region(payload_size, "array payload");
164
314
    auto scope = builder.start_array();
165
671
    while (!payload.empty()) {
166
357
        collect_jsonb_value(payload, builder, relative_depth + 1, base_depth);
167
357
    }
168
314
    scope.finish();
169
314
}
170
171
template <typename Native, typename Builder>
172
void collect_jsonb_decimal(BoundedJsonbCursor& cursor, Builder& builder, uint8_t width,
173
101
                           uint32_t maximum_precision, const char* description) {
174
101
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
101
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
101
    const auto value = cursor.read<Native>("decimal value");
177
101
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
101
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
101
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalIiNS_19VariantBlockBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
20
                           uint32_t maximum_precision, const char* description) {
174
20
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
20
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
20
    const auto value = cursor.read<Native>("decimal value");
177
20
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
20
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
20
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalIlNS_19VariantBlockBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
24
                           uint32_t maximum_precision, const char* description) {
174
24
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
24
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
24
    const auto value = cursor.read<Native>("decimal value");
177
24
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
24
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
24
}
variant_jsonb.cpp:_ZN5doris12_GLOBAL__N_121collect_jsonb_decimalInNS_19VariantBlockBuilder3RowEEEvRNS0_18BoundedJsonbCursorERT0_hjPKc
Line
Count
Source
173
57
                           uint32_t maximum_precision, const char* description) {
174
57
    const auto precision = cursor.read<uint32_t>("decimal precision");
175
57
    const auto scale = cursor.read<uint32_t>("decimal scale");
176
57
    const auto value = cursor.read<Native>("decimal value");
177
57
    require_decimal(value, precision, scale, maximum_precision, 38, description);
178
57
    builder.add_decimal(static_cast<__int128>(value), static_cast<uint8_t>(scale), width);
179
57
}
180
181
template <typename Builder>
182
void collect_jsonb_value(BoundedJsonbCursor& cursor, Builder& builder, uint32_t relative_depth,
183
629
                         uint32_t base_depth) {
184
629
    require_jsonb_depth(relative_depth, false);
185
629
    require_variant_depth(base_depth, relative_depth, false);
186
629
    const auto type = static_cast<JsonbType>(cursor.read<uint8_t>("value type"));
187
629
    switch (type) {
188
15
    case JsonbType::T_Null:
189
15
        builder.add_null();
190
15
        return;
191
5
    case JsonbType::T_True:
192
5
        builder.add_bool(true);
193
5
        return;
194
2
    case JsonbType::T_False:
195
2
        builder.add_bool(false);
196
2
        return;
197
8
    case JsonbType::T_Int8:
198
8
        builder.add_int(cursor.read<int8_t>("int8 value"));
199
8
        return;
200
5
    case JsonbType::T_Int16:
201
5
        builder.add_int(cursor.read<int16_t>("int16 value"));
202
5
        return;
203
7
    case JsonbType::T_Int32:
204
7
        builder.add_int(cursor.read<int32_t>("int32 value"));
205
7
        return;
206
12
    case JsonbType::T_Int64:
207
12
        builder.add_int(cursor.read<int64_t>("int64 value"));
208
12
        return;
209
10
    case JsonbType::T_Double:
210
10
        builder.add_double(cursor.read<double>("double value"));
211
10
        return;
212
17
    case JsonbType::T_String: {
213
17
        const auto size = cursor.read<uint32_t>("string size");
214
17
        builder.add_string(cursor.read_bytes(size, "string payload"));
215
17
        return;
216
0
    }
217
11
    case JsonbType::T_Binary: {
218
11
        const auto size = cursor.read<uint32_t>("binary size");
219
11
        builder.add_binary(cursor.read_bytes(size, "binary payload"));
220
11
        return;
221
0
    }
222
33
    case JsonbType::T_Object:
223
33
        collect_jsonb_object(cursor, builder, relative_depth, base_depth);
224
33
        return;
225
314
    case JsonbType::T_Array:
226
314
        collect_jsonb_array(cursor, builder, relative_depth, base_depth);
227
314
        return;
228
25
    case JsonbType::T_Int128:
229
25
        builder.add_largeint(cursor.read<__int128>("int128 value"));
230
25
        return;
231
6
    case JsonbType::T_Float:
232
6
        builder.add_float(cursor.read<float>("float value"));
233
6
        return;
234
20
    case JsonbType::T_Decimal32:
235
20
        collect_jsonb_decimal<int32_t>(cursor, builder, 4, 9, "Decimal32");
236
20
        return;
237
24
    case JsonbType::T_Decimal64:
238
24
        collect_jsonb_decimal<int64_t>(cursor, builder, 8, 18, "Decimal64");
239
24
        return;
240
57
    case JsonbType::T_Decimal128:
241
57
        collect_jsonb_decimal<__int128>(cursor, builder, 16, 38, "Decimal128");
242
57
        return;
243
41
    case JsonbType::T_Decimal256:
244
41
        throw Exception(ErrorCode::INVALID_ARGUMENT,
245
41
                        "Conversion from JSONB DECIMAL256 to Variant V2 is not supported");
246
0
    case JsonbType::NUM_TYPES:
247
0
        break;
248
629
    }
249
2
    throw Exception(ErrorCode::CORRUPTION, "Unknown JSONB value type {}",
250
2
                    static_cast<uint8_t>(type));
251
629
}
252
253
template <typename Builder>
254
260
void collect_jsonb_document(StringRef document, Builder& builder, uint32_t initial_depth = 0) {
255
260
    if (document.data == nullptr && document.size != 0) {
256
1
        throw Exception(ErrorCode::INVALID_ARGUMENT,
257
1
                        "JSONB input has a null data pointer for {} bytes", document.size);
258
1
    }
259
259
    if (document.size == 0) {
260
2
        builder.add_null();
261
2
        return;
262
2
    }
263
264
257
    BoundedJsonbCursor cursor(document.data, document.size);
265
257
    const auto version = cursor.read<uint8_t>("document version");
266
257
    if (version != JSONB_VER) {
267
2
        throw Exception(ErrorCode::CORRUPTION, "Unsupported JSONB version {}, expected {}", version,
268
2
                        JSONB_VER);
269
2
    }
270
255
    collect_jsonb_value(cursor, builder, 0, initial_depth);
271
255
    if (!cursor.empty()) {
272
1
        throw Exception(ErrorCode::CORRUPTION,
273
1
                        "JSONB document has {} trailing bytes after its root value",
274
1
                        cursor.remaining());
275
1
    }
276
255
}
277
278
433
void require_jsonb_write(bool written, const char* operation) {
279
433
    if (!written) {
280
0
        throw Exception(ErrorCode::INVALID_ARGUMENT, "JSONB writer rejected {}", operation);
281
0
    }
282
433
}
283
284
16
void write_jsonb_string(JsonbWriter& writer, StringRef value, const char* description) {
285
16
    variant_json_detail::require_valid_json_utf8(value, description);
286
16
    require_jsonb_write(writer.writeStartString(), "string start");
287
16
    require_jsonb_write(writer.writeString(value.data, value.size), "string payload");
288
16
    require_jsonb_write(writer.writeEndString(), "string end");
289
16
}
290
291
11
void write_jsonb_string(JsonbWriter& writer, const variant_json_detail::FormattedScalar& value) {
292
11
    write_jsonb_string(writer, {value.bytes.data(), value.size}, "formatted string");
293
11
}
294
295
2
void write_jsonb_binary(JsonbWriter& writer, StringRef value) {
296
2
    require_jsonb_write(writer.writeStartBinary(), "binary start");
297
2
    require_jsonb_write(writer.writeBinary(value.data, value.size), "binary payload");
298
2
    require_jsonb_write(writer.writeEndBinary(), "binary end");
299
2
}
300
301
class VariantToJsonbConverter {
302
public:
303
    VariantToJsonbConverter(JsonbWriter& writer, const VariantJsonFormatOptions& options)
304
26
            : _writer(writer), _options(options) {}
305
306
283
    void write(VariantValueRef value, uint32_t depth) {
307
283
        const VariantBasicType type = value.basic_type();
308
283
        const bool is_container =
309
283
                type == VariantBasicType::OBJECT || type == VariantBasicType::ARRAY;
310
283
        require_jsonb_depth(depth, is_container);
311
283
        switch (type) {
312
4
        case VariantBasicType::SHORT_STRING:
313
4
            write_jsonb_string(_writer, value.get_string(), "Variant string");
314
4
            return;
315
63
        case VariantBasicType::PRIMITIVE:
316
63
            write_primitive(value);
317
63
            return;
318
8
        case VariantBasicType::OBJECT:
319
8
            write_object(value, depth);
320
8
            return;
321
207
        case VariantBasicType::ARRAY:
322
207
            write_array(value, depth);
323
207
            return;
324
283
        }
325
283
    }
326
327
private:
328
12
    void write_decimal(VariantDecimal decimal) {
329
12
        uint32_t precision = 38;
330
12
        if (decimal.width == 4) {
331
4
            precision = 9;
332
8
        } else if (decimal.width == 8) {
333
3
            precision = 18;
334
3
        }
335
12
        require_decimal(decimal.unscaled, precision, decimal.scale, precision, 38,
336
12
                        "Variant decimal");
337
12
        if (decimal.width == 4) {
338
3
            require_jsonb_write(
339
3
                    _writer.writeDecimal(Decimal32 {static_cast<int32_t>(decimal.unscaled)},
340
3
                                         precision, decimal.scale),
341
3
                    "Decimal32 value");
342
9
        } else if (decimal.width == 8) {
343
3
            require_jsonb_write(
344
3
                    _writer.writeDecimal(Decimal64 {static_cast<int64_t>(decimal.unscaled)},
345
3
                                         precision, decimal.scale),
346
3
                    "Decimal64 value");
347
6
        } else {
348
6
            require_jsonb_write(
349
6
                    _writer.writeDecimal(Decimal128V3 {decimal.unscaled}, precision, decimal.scale),
350
6
                    "Decimal128 value");
351
6
        }
352
12
    }
353
354
63
    void write_primitive(VariantValueRef value) {
355
63
        switch (value.primitive_id()) {
356
11
        case VariantPrimitiveId::NULL_VALUE:
357
11
            require_jsonb_write(_writer.writeNull(), "null value");
358
11
            return;
359
4
        case VariantPrimitiveId::TRUE_VALUE:
360
4
            require_jsonb_write(_writer.writeBool(true), "true value");
361
4
            return;
362
3
        case VariantPrimitiveId::FALSE_VALUE:
363
3
            require_jsonb_write(_writer.writeBool(false), "false value");
364
3
            return;
365
8
        case VariantPrimitiveId::INT8:
366
8
            require_jsonb_write(_writer.writeInt8(static_cast<int8_t>(value.get_int())),
367
8
                                "int8 value");
368
8
            return;
369
2
        case VariantPrimitiveId::INT16:
370
2
            require_jsonb_write(_writer.writeInt16(static_cast<int16_t>(value.get_int())),
371
2
                                "int16 value");
372
2
            return;
373
2
        case VariantPrimitiveId::INT32:
374
2
            require_jsonb_write(_writer.writeInt32(static_cast<int32_t>(value.get_int())),
375
2
                                "int32 value");
376
2
            return;
377
2
        case VariantPrimitiveId::INT64:
378
2
            require_jsonb_write(_writer.writeInt64(value.get_int()), "int64 value");
379
2
            return;
380
2
        case VariantPrimitiveId::DOUBLE:
381
2
            require_jsonb_write(_writer.writeDouble(value.get_double()), "double value");
382
2
            return;
383
4
        case VariantPrimitiveId::DECIMAL4:
384
7
        case VariantPrimitiveId::DECIMAL8:
385
12
        case VariantPrimitiveId::DECIMAL16:
386
12
            write_decimal(value.get_decimal());
387
12
            return;
388
1
        case VariantPrimitiveId::DATE:
389
1
            write_jsonb_string(_writer, variant_json_detail::format_json_date(value.get_date()));
390
1
            return;
391
2
        case VariantPrimitiveId::TIMESTAMP_MICROS:
392
2
            write_jsonb_string(_writer,
393
2
                               variant_json_detail::format_json_timestamp(
394
2
                                       value.get_timestamp_micros(), 6, true, _options.timezone));
395
2
            return;
396
2
        case VariantPrimitiveId::TIMESTAMP_NTZ_MICROS:
397
2
            write_jsonb_string(_writer,
398
2
                               variant_json_detail::format_json_timestamp(
399
2
                                       value.get_timestamp_ntz_micros(), 6, false, nullptr));
400
2
            return;
401
2
        case VariantPrimitiveId::FLOAT:
402
2
            require_jsonb_write(_writer.writeFloat(value.get_float()), "float value");
403
2
            return;
404
2
        case VariantPrimitiveId::BINARY:
405
2
            write_jsonb_binary(_writer, value.get_binary());
406
2
            return;
407
1
        case VariantPrimitiveId::STRING:
408
1
            write_jsonb_string(_writer, value.get_string(), "Variant string");
409
1
            return;
410
2
        case VariantPrimitiveId::TIME_NTZ_MICROS:
411
2
            write_jsonb_string(_writer, variant_json_detail::format_json_time_micros(
412
2
                                                value.get_time_ntz_micros()));
413
2
            return;
414
2
        case VariantPrimitiveId::TIMESTAMP_NANOS:
415
2
            write_jsonb_string(_writer,
416
2
                               variant_json_detail::format_json_timestamp(
417
2
                                       value.get_timestamp_nanos(), 9, true, _options.timezone));
418
2
            return;
419
2
        case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS:
420
2
            write_jsonb_string(_writer,
421
2
                               variant_json_detail::format_json_timestamp(
422
2
                                       value.get_timestamp_ntz_nanos(), 9, false, nullptr));
423
2
            return;
424
1
        case VariantPrimitiveId::UUID:
425
1
            write_jsonb_string(_writer, variant_json_detail::format_json_uuid(value.get_uuid()));
426
1
            return;
427
63
        }
428
0
        throw Exception(ErrorCode::CORRUPTION, "Unknown Variant primitive id");
429
63
    }
430
431
8
    void write_object(VariantValueRef value, uint32_t depth) {
432
8
        require_jsonb_write(_writer.writeStartObject(), "object start");
433
8
        const uint32_t count = value.num_elements();
434
8
        StringRef previous_key;
435
16
        for (uint32_t index = 0; index < count; ++index) {
436
11
            uint32_t field_id = 0;
437
11
            const VariantValueRef child = value.object_value_at(index, &field_id);
438
11
            const StringRef key = value.metadata.key_at(field_id);
439
11
            variant_json_detail::require_valid_json_utf8(key, "Variant object key");
440
11
            if (key.size > std::numeric_limits<uint8_t>::max()) {
441
1
                throw Exception(ErrorCode::INVALID_ARGUMENT,
442
1
                                "Variant object key length {} exceeds JSONB maximum {}", key.size,
443
1
                                std::numeric_limits<uint8_t>::max());
444
1
            }
445
10
            if (index != 0 && previous_key.compare(key) >= 0) {
446
2
                throw Exception(ErrorCode::CORRUPTION,
447
2
                                "Variant object keys are not strictly byte-sorted at field {}",
448
2
                                index);
449
2
            }
450
8
            require_jsonb_write(_writer.writeKey(key.data, static_cast<uint8_t>(key.size)),
451
8
                                "object key");
452
8
            write(child, depth + 1);
453
8
            previous_key = key;
454
8
        }
455
5
        require_jsonb_write(_writer.writeEndObject(), "object end");
456
5
    }
457
458
207
    void write_array(VariantValueRef value, uint32_t depth) {
459
207
        require_jsonb_write(_writer.writeStartArray(), "array start");
460
207
        const uint32_t count = value.num_elements();
461
456
        for (uint32_t index = 0; index < count; ++index) {
462
249
            write(value.array_at(index), depth + 1);
463
249
        }
464
207
        require_jsonb_write(_writer.writeEndArray(), "array end");
465
207
    }
466
467
    JsonbWriter& _writer;
468
    const VariantJsonFormatOptions& _options;
469
};
470
471
} // namespace
472
473
struct JsonbToVariantEncoder::Impl {
474
    enum class State : uint8_t { COLLECTING, FINISHED, FAILED };
475
476
16
    explicit Impl(VariantBlockBuilder::ReserveHint hint) : builder(hint) {}
477
478
35
    void require_collecting() const {
479
35
        if (state == State::FINISHED) {
480
2
            throw Exception(ErrorCode::INVALID_ARGUMENT,
481
2
                            "Variant JSONB encoder is already finished");
482
2
        }
483
33
        if (state == State::FAILED) {
484
3
            throw Exception(ErrorCode::INVALID_ARGUMENT,
485
3
                            "Variant JSONB encoder is in a terminal failed state");
486
3
        }
487
33
    }
488
489
    VariantBlockBuilder builder;
490
    State state = State::COLLECTING;
491
};
492
493
JsonbToVariantEncoder::JsonbToVariantEncoder()
494
11
        : JsonbToVariantEncoder(VariantBlockBuilder::ReserveHint {}) {}
495
JsonbToVariantEncoder::JsonbToVariantEncoder(VariantBlockBuilder::ReserveHint hint)
496
16
        : _impl(std::make_unique<Impl>(hint)) {}
497
16
JsonbToVariantEncoder::~JsonbToVariantEncoder() = default;
498
0
JsonbToVariantEncoder::JsonbToVariantEncoder(JsonbToVariantEncoder&&) noexcept = default;
499
0
JsonbToVariantEncoder& JsonbToVariantEncoder::operator=(JsonbToVariantEncoder&&) noexcept = default;
500
501
2
void JsonbToVariantEncoder::add_null() {
502
2
    _impl->require_collecting();
503
2
    try {
504
2
        auto row = _impl->builder.begin_row();
505
2
        row.add_null();
506
2
        row.finish();
507
2
    } catch (...) {
508
0
        _impl->state = Impl::State::FAILED;
509
0
        throw;
510
0
    }
511
2
}
512
513
18
void JsonbToVariantEncoder::add_jsonb(StringRef document) {
514
18
    _impl->require_collecting();
515
18
    try {
516
18
        auto row = _impl->builder.begin_row();
517
18
        collect_jsonb_document(document, row);
518
18
        row.finish();
519
18
    } catch (...) {
520
4
        _impl->state = Impl::State::FAILED;
521
4
        throw;
522
4
    }
523
18
}
524
525
15
VariantEncodedBlock JsonbToVariantEncoder::finish_block() {
526
15
    _impl->require_collecting();
527
15
    try {
528
15
        VariantEncodedBlock block = _impl->builder.finish_block();
529
15
        _impl->state = Impl::State::FINISHED;
530
15
        return block;
531
15
    } catch (...) {
532
0
        _impl->state = Impl::State::FAILED;
533
0
        throw;
534
0
    }
535
15
}
536
537
244
void jsonb_to_variant(StringRef document, VariantBlockBuilder::Row& row, uint32_t initial_depth) {
538
244
    try {
539
244
        collect_jsonb_document(document, row, initial_depth);
540
244
    } catch (...) {
541
231
        row.abort();
542
231
        throw;
543
231
    }
544
244
}
545
546
void variant_to_jsonb(VariantValueRef value, JsonbWriter& writer,
547
29
                      const VariantJsonFormatOptions& options) {
548
29
    writer.reset();
549
29
    try {
550
29
        if (value.metadata.version() != VARIANT_ENCODING_VERSION) {
551
0
            throw Exception(ErrorCode::CORRUPTION, "Unsupported Variant metadata version {}",
552
0
                            value.metadata.version());
553
0
        }
554
29
        static_cast<void>(value.metadata.dict_size());
555
29
        variant_json_detail::require_exact_json_value(value);
556
29
        VariantToJsonbConverter(writer, options).write(value, 0);
557
29
    } catch (...) {
558
10
        writer.reset();
559
10
        throw;
560
10
    }
561
29
}
562
563
} // namespace doris