be/src/util/jsonb_parser_simd.h

Source
/*
 *  Copyright (c) 2014, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

/*
 * This file defines JsonbParserTSIMD (template) and JsonbParser.
 *
 * JsonbParserTSIMD is a template class which implements a JSON parser.
 * JsonbParserTSIMD parses JSON text, and serialize it to JSONB binary format
 * by using JsonbWriterT object. By default, JsonbParserTSIMD creates a new
 * JsonbWriterT object with an output stream object.  However, you can also
 * pass in your JsonbWriterT or any stream object that implements some basic
 * interface of std::ostream (see JsonbStream.h).
 *
 * JsonbParser specializes JsonbParserTSIMD with JsonbOutStream type (see
 * JsonbStream.h). So unless you want to provide own a different output stream
 * type, use JsonbParser object.
 *
 * ** Parsing JSON **
 * JsonbParserTSIMD parses JSON string, and directly serializes into JSONB
 * packed bytes. There are three ways to parse a JSON string: (1) using
 * c-string, (2) using string with len, (3) using std::istream object. You can
 * use custom streambuf to redirect output. JsonbOutBuffer is a streambuf used
 * internally if the input is raw character buffer.
 *
 * You can reuse an JsonbParserTSIMD object to parse/serialize multiple JSON
 * strings, and the previous JSONB will be overwritten.
 *
 * If parsing fails (returned false), the error code will be set to one of
 * JsonbErrType, and can be retrieved by calling getErrorCode().
 *
 * ** External dictionary **
 * During parsing a JSON string, you can pass a call-back function to map a key
 * string to an id, and store the dictionary id in JSONB to save space. The
 * purpose of using an external dictionary is more towards a collection of
 * documents (which has common keys) rather than a single document, so that
 * space saving will be significant.
 *
 * ** Endianness **
 * Note: JSONB serialization doesn't assume endianness of the server. However
 * you will need to ensure that the endianness at the reader side is the same
 * as that at the writer side (if they are on different machines). Otherwise,
 * proper conversion is needed when a number value is returned to the
 * caller/writer.
 *
 * @author Tian Xia <tianx@fb.com>
 * 
 * this file is copied from 
 * https://github.com/facebook/mysql-5.6/blob/fb-mysql-5.6.35/fbson/FbsonJsonParser.h
 * and modified by Doris
 */

#pragma once
#include <simdjson.h>

#include <cmath>
#include <limits>
#include <string>
#include <string_view>

#include "common/status.h"
#include "util/jsonb_document.h"
#include "util/jsonb_writer.h"
#include "util/string_parser.hpp"

namespace doris {
using int128_t = __int128;
struct JsonbParser {
    // parse a UTF-8 JSON string with length
    // will reset writer before parse
    static Status parse(const char* pch, size_t len, JsonbWriter& writer) {
        if (!pch || len == 0) {
            return Status::InvalidArgument("Empty JSON document");
        }
        writer.reset();
        try {
            simdjson::ondemand::parser simdjson_parser;
            simdjson::padded_string json_str {pch, len};
            simdjson::ondemand::document doc = simdjson_parser.iterate(json_str);

            // simdjson process top level primitive types specially
            // so some repeated code here
            switch (doc.type()) {
            case simdjson::ondemand::json_type::object:
            case simdjson::ondemand::json_type::array: {
                RETURN_IF_ERROR(parse(doc.get_value(), writer));
                break;
            }
            case simdjson::ondemand::json_type::null: {
                if (writer.writeNull() == 0) {
                    return Status::InvalidArgument("writeNull failed");
                }
                break;
            }
            case simdjson::ondemand::json_type::boolean: {
                if (writer.writeBool(doc.get_bool()) == 0) {
                    return Status::InvalidArgument("writeBool failed");
                }
                break;
            }
            case simdjson::ondemand::json_type::string: {
                RETURN_IF_ERROR(write_string(doc.get_string(), writer));
                break;
            }
            case simdjson::ondemand::json_type::number: {
                simdjson::ondemand::number num;
                simdjson::error_code res = doc.get_number().get(num);
                std::string_view token = doc.raw_json_token();
                // For a root number simdjson reports NUMBER_ERROR / BIGINT_ERROR before it
                // checks for trailing content, and the raw token stops at the next token, so
                // `18446744073709551616 0` would otherwise be accepted as its first token.
                // A root number must reach the end of the document.
                if (token.data() + token.size() != json_str.data() + json_str.size()) {
                    return Status::InvalidArgument(
                            "simdjson get_number failed: trailing content after root number "
                            "{}",
                            quote_token(token));
                }
                RETURN_IF_ERROR(write_number(res, num, token, writer));
                break;
            }
            }
        } catch (simdjson::simdjson_error& e) {
            return Status::InvalidArgument(fmt::format("simdjson parse exception: {}", e.what()));
        }
        return Status::OK();
    }

    // Error messages quote the offending input so that the bad value can be located, but
    // the input may be a multi-megabyte value (e.g. a malformed digit run) and tolerant
    // callers such as json_valid, the error-to-null variants or non-strict CAST discard the
    // message right away. Keep the quoted part bounded and report the full length instead.
    static std::string bounded_quote(std::string_view text) {
        constexpr size_t kMaxQuotedLen = 64;
        if (text.size() <= kMaxQuotedLen) {
            return std::string(text);
        }
        return fmt::format("{}... (truncated, {} bytes)", text.substr(0, kMaxQuotedLen),
                           text.size());
    }

private:
    // parse json, recursively if necessary, by simdjson
    //  and serialize to binary format by writer
    static Status parse(simdjson::ondemand::value value, JsonbWriter& writer) {
        switch (value.type()) {
        case simdjson::ondemand::json_type::null: {
            if (writer.writeNull() == 0) {
                return Status::InvalidArgument("writeNull failed");
            }
            break;
        }
        case simdjson::ondemand::json_type::boolean: {
            if (writer.writeBool(value.get_bool()) == 0) {
                return Status::InvalidArgument("writeBool failed");
            }
            break;
        }
        case simdjson::ondemand::json_type::string: {
            RETURN_IF_ERROR(write_string(value.get_string(), writer));
            break;
        }
        case simdjson::ondemand::json_type::number: {
            simdjson::ondemand::number num;
            simdjson::error_code res = value.get_number().get(num);
            RETURN_IF_ERROR(write_number(res, num, value.raw_json_token(), writer));
            break;
        }
        case simdjson::ondemand::json_type::object: {
            if (!writer.writeStartObject()) {
                return Status::InvalidArgument("writeStartObject failed");
            }

            for (auto kv : value.get_object()) {
                std::string_view key;
                simdjson::error_code e = kv.unescaped_key().get(key);
                if (e != simdjson::SUCCESS) {
                    return Status::InvalidArgument(fmt::format("simdjson get key failed: {}", e));
                }

                // write key
                if (key.size() > std::numeric_limits<uint8_t>::max()) {
                    return Status::InvalidArgument("key size exceeds max limit: {} , {}",
                                                   key.size(), std::numeric_limits<uint8_t>::max());
                }
                if (!writer.writeKey(key.data(), (uint8_t)key.size())) {
                    return Status::InvalidArgument("writeKey failed : {}", key);
                }

                // parse object value
                RETURN_IF_ERROR(parse(kv.value(), writer));
            }

            if (!writer.writeEndObject()) {
                return Status::InvalidArgument("writeEndObject failed");
                break;
            }

            break;
        }
        case simdjson::ondemand::json_type::array: {
            if (!writer.writeStartArray()) {
                return Status::InvalidArgument("writeStartArray failed");
            }

            for (auto elem : value.get_array()) {
                // parse array element
                RETURN_IF_ERROR(parse(elem.value(), writer));
            }

            if (!writer.writeEndArray()) {
                return Status::InvalidArgument("writeEndArray failed");
            }
            break;
        }
        default: {
            return Status::InvalidArgument("unknown value type: ");
        }

        } // end of switch
        return Status::OK();
    }

    static Status write_string(std::string_view str, JsonbWriter& writer) {
        // start writing string
        if (!writer.writeStartString()) {
            return Status::InvalidArgument("writeStartString failed");
        }

        // write string
        if (str.size() > 0) {
            if (writer.writeString(str.data(), str.size()) == 0) {
                return Status::InvalidArgument("writeString failed");
            }
        }

        // end writing string
        if (!writer.writeEndString()) {
            return Status::InvalidArgument("writeEndString failed");
        }
        return Status::OK();
    }

    // raw_json_token() spans up to the start of the next token, so it may end with JSON
    // whitespace that is not part of the number.
    static std::string_view trim_trailing_whitespace(std::string_view token) {
        while (!token.empty() && (token.back() == ' ' || token.back() == '\t' ||
                                  token.back() == '\n' || token.back() == '\r')) {
            token.remove_suffix(1);
        }
        return token;
    }

    static std::string quote_token(std::string_view token) {
        return bounded_quote(trim_trailing_whitespace(token));
    }

    // Matches the JSON number grammar exactly:
    //   -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
    static bool is_json_number(std::string_view token) {
        size_t i = 0;
        const size_t n = token.size();
        auto skip_digits = [&]() {
            const size_t start = i;
            while (i < n && token[i] >= '0' && token[i] <= '9') {
                ++i;
            }
            return i > start;
        };
        if (i < n && token[i] == '-') {
            ++i;
        }
        if (i < n && token[i] == '0') {
            ++i;
        } else if (!skip_digits()) {
            return false;
        }
        if (i < n && token[i] == '.') {
            ++i;
            if (!skip_digits()) {
                return false;
            }
        }
        if (i < n && (token[i] == 'e' || token[i] == 'E')) {
            ++i;
            if (i < n && (token[i] == '+' || token[i] == '-')) {
                ++i;
            }
            if (!skip_digits()) {
                return false;
            }
        }
        return i == n;
    }

    // According to https://github.com/simdjson/simdjson/pull/2139, integers that do not fit
    // in 64 bits can be handled by parsing the raw_json_token ourselves: simdjson returns
    // NUMBER_ERROR for 18446744073709551616 (one above uint64 max) and BIGINT_ERROR for
    // longer integers such as 18446744073709551616231231.
    // However NUMBER_ERROR is also what simdjson returns for malformed tokens (leading
    // zeros like 01, a trailing dot like 1., 1e, trailing garbage like 1x) and for values
    // beyond the double range. `num` carries nothing usable in any of these cases, so the
    // raw token is first checked against the JSON number grammar and then parsed as int128
    // or double.
    static Status write_number_from_token(simdjson::error_code res, std::string_view raw_string,
                                          JsonbWriter& writer) {
        std::string_view token = trim_trailing_whitespace(raw_string);
        if (!is_json_number(token)) {
            return Status::InvalidArgument("simdjson get_number failed: {}, raw string is: {}",
                                           simdjson::error_message(res), quote_token(token));
        }

        // StringParser::string_to_int silently truncates a fraction, so only a token made of
        // digits may be parsed as an integer.
        if (token.find_first_of(".eE") == std::string_view::npos) {
            StringParser::ParseResult result;
            auto val = StringParser::string_to_int<int128_t>(token.data(), token.size(), &result);
            if (result == StringParser::PARSE_SUCCESS) {
                if (!writer.writeInt128(val)) {
                    return Status::InvalidArgument("writeInt128 failed");
                }
                return Status::OK();
            }
        }

        // Either a floating point number or an integer beyond int128. Converting it to double
        // may lose precision, but for JSON, exchanging data as plain text between different
        // systems may inherently cause precision loss.
        StringParser::ParseResult result;
        double double_val =
                StringParser::string_to_float<double>(token.data(), token.size(), &result);
        if (result != StringParser::PARSE_SUCCESS || !std::isfinite(double_val)) {
            return Status::InvalidArgument("invalid number, raw string is: {}", quote_token(token));
        }
        if (!writer.writeDouble(double_val)) {
            return Status::InvalidArgument("writeDouble failed");
        }
        return Status::OK();
    }

    static Status write_number(simdjson::error_code res, simdjson::ondemand::number num,
                               std::string_view raw_string, JsonbWriter& writer) {
        switch (res) {
        case simdjson::error_code::SUCCESS:
            break;
        case simdjson::error_code::NUMBER_ERROR:
        case simdjson::error_code::BIGINT_ERROR:
            return write_number_from_token(res, raw_string, writer);
        default:
            // simdjson reports no other error for a number token (a root number followed by
            // another token is already rejected by the end-of-document check in parse()), so
            // anything else is reported as is.
            return Status::InvalidArgument("simdjson get_number failed: {}, raw string is: {}",
                                           simdjson::error_message(res), quote_token(raw_string));
        }

        // On success simdjson yields one of three number types:
        // 1. floating_point_number: A binary64 number, which will be converted to jsonb's double type.
        // 2. signed_integer: A signed integer that fits in a 64-bit word using two's complement.
        // 3. unsigned_integer: A positive integer larger or equal to 1<<63.
        //    For these two integer types, we will convert them to jsonb's int8/int16/int32/int64/int128 types according to the specific value.
        switch (num.get_number_type()) {
        case simdjson::ondemand::number_type::floating_point_number: {
            if (writer.writeDouble(num.get_double()) == 0) {
                return Status::InvalidArgument("writeDouble failed");
            }
            break;
        }
        case simdjson::ondemand::number_type::signed_integer:
        case simdjson::ondemand::number_type::unsigned_integer: {
            int128_t val = num.is_int64() ? (int128_t)num.get_int64() : (int128_t)num.get_uint64();
            bool success = false;
            if (val >= std::numeric_limits<int8_t>::min() &&
                val <= std::numeric_limits<int8_t>::max()) {
                success = writer.writeInt8((int8_t)val);
            } else if (val >= std::numeric_limits<int16_t>::min() &&
                       val <= std::numeric_limits<int16_t>::max()) {
                success = writer.writeInt16((int16_t)val);
            } else if (val >= std::numeric_limits<int32_t>::min() &&
                       val <= std::numeric_limits<int32_t>::max()) {
                success = writer.writeInt32((int32_t)val);
            } else if (val >= std::numeric_limits<int64_t>::min() &&
                       val <= std::numeric_limits<int64_t>::max()) {
                success = writer.writeInt64((int64_t)val);
            } else { // INT128
                success = writer.writeInt128(val);
            }

            if (!success) {
                return Status::InvalidArgument("writeInt failed");
            }
            break;
        }
        case simdjson::ondemand::number_type::big_integer: {
            // simdjson never parses a big_integer successfully; integers beyond 64 bits
            // arrive as NUMBER_ERROR / BIGINT_ERROR and are handled by
            // write_number_from_token above.
            __builtin_unreachable();
        }
        }
        return Status::OK();
    }
};
} // namespace doris

Coverage Report

Created: 2026-09-25 19:11

Line	Count	Source
1		/*
2		* Copyright (c) 2014, Facebook, Inc.
3		* All rights reserved.
4		*
5		* This source code is licensed under the BSD-style license found in the
6		* LICENSE file in the root directory of this source tree. An additional grant
7		* of patent rights can be found in the PATENTS file in the same directory.
8		*
9		*/
10
11		/*
12		* This file defines JsonbParserTSIMD (template) and JsonbParser.
13		*
14		* JsonbParserTSIMD is a template class which implements a JSON parser.
15		* JsonbParserTSIMD parses JSON text, and serialize it to JSONB binary format
16		* by using JsonbWriterT object. By default, JsonbParserTSIMD creates a new
17		* JsonbWriterT object with an output stream object. However, you can also
18		* pass in your JsonbWriterT or any stream object that implements some basic
19		* interface of std::ostream (see JsonbStream.h).
20		*
21		* JsonbParser specializes JsonbParserTSIMD with JsonbOutStream type (see
22		* JsonbStream.h). So unless you want to provide own a different output stream
23		* type, use JsonbParser object.
24		*
25		* Parsing JSON
26		* JsonbParserTSIMD parses JSON string, and directly serializes into JSONB
27		* packed bytes. There are three ways to parse a JSON string: (1) using
28		* c-string, (2) using string with len, (3) using std::istream object. You can
29		* use custom streambuf to redirect output. JsonbOutBuffer is a streambuf used
30		* internally if the input is raw character buffer.
31		*
32		* You can reuse an JsonbParserTSIMD object to parse/serialize multiple JSON
33		* strings, and the previous JSONB will be overwritten.
34		*
35		* If parsing fails (returned false), the error code will be set to one of
36		* JsonbErrType, and can be retrieved by calling getErrorCode().
37		*
38		* External dictionary
39		* During parsing a JSON string, you can pass a call-back function to map a key
40		* string to an id, and store the dictionary id in JSONB to save space. The
41		* purpose of using an external dictionary is more towards a collection of
42		* documents (which has common keys) rather than a single document, so that
43		* space saving will be significant.
44		*
45		* Endianness
46		* Note: JSONB serialization doesn't assume endianness of the server. However
47		* you will need to ensure that the endianness at the reader side is the same
48		* as that at the writer side (if they are on different machines). Otherwise,
49		* proper conversion is needed when a number value is returned to the
50		* caller/writer.
51		*
52		* @author Tian Xia <tianx@fb.com>
53		*
54		* this file is copied from
55		* https://github.com/facebook/mysql-5.6/blob/fb-mysql-5.6.35/fbson/FbsonJsonParser.h
56		* and modified by Doris
57		*/
58
59		#pragma once
60		#include <simdjson.h>
61
62		#include <cmath>
63		#include <limits>
64		#include <string>
65		#include <string_view>
66
67		#include "common/status.h"
68		#include "util/jsonb_document.h"
69		#include "util/jsonb_writer.h"
70		#include "util/string_parser.hpp"
71
72		namespace doris {
73		using int128_t = __int128;
74		struct JsonbParser {
75		// parse a UTF-8 JSON string with length
76		// will reset writer before parse
77	6.26k	static Status parse(const char* pch, size_t len, JsonbWriter& writer) {
78	6.26k	if (!pch \|\| len == 0) {
79	11	return Status::InvalidArgument("Empty JSON document");
80	11	}
81	6.25k	writer.reset();
82	6.25k	try {
83	6.25k	simdjson::ondemand::parser simdjson_parser;
84	6.25k	simdjson::padded_string json_str {pch, len};
85	6.25k	simdjson::ondemand::document doc = simdjson_parser.iterate(json_str);
86
87		// simdjson process top level primitive types specially
88		// so some repeated code here
89	6.25k	switch (doc.type()) {
90	3.88k	case simdjson::ondemand::json_type::object:
91	5.46k	case simdjson::ondemand::json_type::array: {
92	5.46k	RETURN_IF_ERROR(parse(doc.get_value(), writer));
93	5.17k	break;
94	5.46k	}
95	5.17k	case simdjson::ondemand::json_type::null: {
96	56	if (writer.writeNull() == 0) {
97	0	return Status::InvalidArgument("writeNull failed");
98	0	}
99	56	break;
100	56	}
101	71	case simdjson::ondemand::json_type::boolean: {
102	71	if (writer.writeBool(doc.get_bool()) == 0) {
103	0	return Status::InvalidArgument("writeBool failed");
104	0	}
105	71	break;
106	71	}
107	324	case simdjson::ondemand::json_type::string: {
108	324	RETURN_IF_ERROR(write_string(doc.get_string(), writer));
109	324	break;
110	324	}
111	324	case simdjson::ondemand::json_type::number: {
112	324	simdjson::ondemand::number num;
113	324	simdjson::error_code res = doc.get_number().get(num);
114	324	std::string_view token = doc.raw_json_token();
115		// For a root number simdjson reports NUMBER_ERROR / BIGINT_ERROR before it
116		// checks for trailing content, and the raw token stops at the next token, so
117		// `18446744073709551616 0` would otherwise be accepted as its first token.
118		// A root number must reach the end of the document.
119	324	if (token.data() + token.size() != json_str.data() + json_str.size()) {
120	10	return Status::InvalidArgument(
121	10	"simdjson get_number failed: trailing content after root number "
122	10	"{}",
123	10	quote_token(token));
124	10	}
125	314	RETURN_IF_ERROR(write_number(res, num, token, writer));
126	284	break;
127	314	}
128	6.25k	}
129	6.25k	} catch (simdjson::simdjson_error& e) {
130	28	return Status::InvalidArgument(fmt::format("simdjson parse exception: {}", e.what()));
131	28	}
132	5.89k	return Status::OK();
133	6.25k	}
134
135		// Error messages quote the offending input so that the bad value can be located, but
136		// the input may be a multi-megabyte value (e.g. a malformed digit run) and tolerant
137		// callers such as json_valid, the error-to-null variants or non-strict CAST discard the
138		// message right away. Keep the quoted part bounded and report the full length instead.
139	62	static std::string bounded_quote(std::string_view text) {
140	62	constexpr size_t kMaxQuotedLen = 64;
141	62	if (text.size() <= kMaxQuotedLen) {
142	54	return std::string(text);
143	54	}
144	8	return fmt::format("{}... (truncated, {} bytes)", text.substr(0, kMaxQuotedLen),
145	8	text.size());
146	62	}
147
148		private:
149		// parse json, recursively if necessary, by simdjson
150		// and serialize to binary format by writer
151	570k	static Status parse(simdjson::ondemand::value value, JsonbWriter& writer) {
152	570k	switch (value.type()) {
153	11.7k	case simdjson::ondemand::json_type::null: {
154	11.7k	if (writer.writeNull() == 0) {
155	0	return Status::InvalidArgument("writeNull failed");
156	0	}
157	11.7k	break;
158	11.7k	}
159	20.5k	case simdjson::ondemand::json_type::boolean: {
160	20.5k	if (writer.writeBool(value.get_bool()) == 0) {
161	0	return Status::InvalidArgument("writeBool failed");
162	0	}
163	20.5k	break;
164	20.5k	}
165	250k	case simdjson::ondemand::json_type::string: {
166	250k	RETURN_IF_ERROR(write_string(value.get_string(), writer));
167	250k	break;
168	250k	}
169	250k	case simdjson::ondemand::json_type::number: {
170	227k	simdjson::ondemand::number num;
171	227k	simdjson::error_code res = value.get_number().get(num);
172	227k	RETURN_IF_ERROR(write_number(res, num, value.raw_json_token(), writer));
173	227k	break;
174	227k	}
175	227k	case simdjson::ondemand::json_type::object: {
176	7.45k	if (!writer.writeStartObject()) {
177	0	return Status::InvalidArgument("writeStartObject failed");
178	0	}
179
180	11.2k	for (auto kv : value.get_object()) {
181	11.2k	std::string_view key;
182	11.2k	simdjson::error_code e = kv.unescaped_key().get(key);
183	11.2k	if (e != simdjson::SUCCESS) {
184	2	return Status::InvalidArgument(fmt::format("simdjson get key failed: {}", e));
185	2	}
186
187		// write key
188	11.2k	if (key.size() > std::numeric_limits<uint8_t>::max()) {
189	268	return Status::InvalidArgument("key size exceeds max limit: {} , {}",
190	268	key.size(), std::numeric_limits<uint8_t>::max());
191	268	}
192	10.9k	if (!writer.writeKey(key.data(), (uint8_t)key.size())) {
193	0	return Status::InvalidArgument("writeKey failed : {}", key);
194	0	}
195
196		// parse object value
197	10.9k	RETURN_IF_ERROR(parse(kv.value(), writer));
198	10.9k	}
199
200	7.09k	if (!writer.writeEndObject()) {
201	0	return Status::InvalidArgument("writeEndObject failed");
202	0	break;
203	0	}
204
205	7.09k	break;
206	7.09k	}
207	51.8k	case simdjson::ondemand::json_type::array: {
208	51.8k	if (!writer.writeStartArray()) {
209	0	return Status::InvalidArgument("writeStartArray failed");
210	0	}
211
212	553k	for (auto elem : value.get_array()) {
213		// parse array element
214	553k	RETURN_IF_ERROR(parse(elem.value(), writer));
215	553k	}
216
217	51.6k	if (!writer.writeEndArray()) {
218	0	return Status::InvalidArgument("writeEndArray failed");
219	0	}
220	51.6k	break;
221	51.6k	}
222	51.6k	default: {
223	0	return Status::InvalidArgument("unknown value type: ");
224	51.6k	}
225
226	570k	} // end of switch
227	569k	return Status::OK();
228	570k	}
229
230	250k	static Status write_string(std::string_view str, JsonbWriter& writer) {
231		// start writing string
232	250k	if (!writer.writeStartString()) {
233	0	return Status::InvalidArgument("writeStartString failed");
234	0	}
235
236		// write string
237	250k	if (str.size() > 0) {
238	239k	if (writer.writeString(str.data(), str.size()) == 0) {
239	0	return Status::InvalidArgument("writeString failed");
240	0	}
241	239k	}
242
243		// end writing string
244	250k	if (!writer.writeEndString()) {
245	0	return Status::InvalidArgument("writeEndString failed");
246	0	}
247	250k	return Status::OK();
248	250k	}
249
250		// raw_json_token() spans up to the start of the next token, so it may end with JSON
251		// whitespace that is not part of the number.
252	123	static std::string_view trim_trailing_whitespace(std::string_view token) {
253	136	while (!token.empty() && (token.back() == ' ' \|\| token.back() == '\t' \|\|
254	136	token.back() == '\n' \|\| token.back() == '\r')) {
255	13	token.remove_suffix(1);
256	13	}
257	123	return token;
258	123	}
259
260	58	static std::string quote_token(std::string_view token) {
261	58	return bounded_quote(trim_trailing_whitespace(token));
262	58	}
263
264		// Matches the JSON number grammar exactly:
265		// -?(0\|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
266	65	static bool is_json_number(std::string_view token) {
267	65	size_t i = 0;
268	65	const size_t n = token.size();
269	89	auto skip_digits = [&]() {
270	89	const size_t start = i;
271	4.19M	while (i < n && token[i] >= '0' && token[i] <= '9') {
272	4.19M	++i;
273	4.19M	}
274	89	return i > start;
275	89	};
276	65	if (i < n && token[i] == '-') {
277	10	++i;
278	10	}
279	65	if (i < n && token[i] == '0') {
280	9	++i;
281	56	} else if (!skip_digits()) {
282	2	return false;
283	2	}
284	63	if (i < n && token[i] == '.') {
285	20	++i;
286	20	if (!skip_digits()) {
287	9	return false;
288	9	}
289	20	}
290	54	if (i < n && (token[i] == 'e' \|\| token[i] == 'E')) {
291	13	++i;
292	13	if (i < n && (token[i] == '+' \|\| token[i] == '-')) {
293	3	++i;
294	3	}
295	13	if (!skip_digits()) {
296	7	return false;
297	7	}
298	13	}
299	47	return i == n;
300	54	}
301
302		// According to https://github.com/simdjson/simdjson/pull/2139, integers that do not fit
303		// in 64 bits can be handled by parsing the raw_json_token ourselves: simdjson returns
304		// NUMBER_ERROR for 18446744073709551616 (one above uint64 max) and BIGINT_ERROR for
305		// longer integers such as 18446744073709551616231231.
306		// However NUMBER_ERROR is also what simdjson returns for malformed tokens (leading
307		// zeros like 01, a trailing dot like 1., 1e, trailing garbage like 1x) and for values
308		// beyond the double range. `num` carries nothing usable in any of these cases, so the
309		// raw token is first checked against the JSON number grammar and then parsed as int128
310		// or double.
311		static Status write_number_from_token(simdjson::error_code res, std::string_view raw_string,
312	65	JsonbWriter& writer) {
313	65	std::string_view token = trim_trailing_whitespace(raw_string);
314	65	if (!is_json_number(token)) {
315	41	return Status::InvalidArgument("simdjson get_number failed: {}, raw string is: {}",
316	41	simdjson::error_message(res), quote_token(token));
317	41	}
318
319		// StringParser::string_to_int silently truncates a fraction, so only a token made of
320		// digits may be parsed as an integer.
321	24	if (token.find_first_of(".eE") == std::string_view::npos) {
322	17	StringParser::ParseResult result;
323	17	auto val = StringParser::string_to_int<int128_t>(token.data(), token.size(), &result);
324	17	if (result == StringParser::PARSE_SUCCESS) {
325	10	if (!writer.writeInt128(val)) {
326	0	return Status::InvalidArgument("writeInt128 failed");
327	0	}
328	10	return Status::OK();
329	10	}
330	17	}
331
332		// Either a floating point number or an integer beyond int128. Converting it to double
333		// may lose precision, but for JSON, exchanging data as plain text between different
334		// systems may inherently cause precision loss.
335	14	StringParser::ParseResult result;
336	14	double double_val =
337	14	StringParser::string_to_float<double>(token.data(), token.size(), &result);
338	14	if (result != StringParser::PARSE_SUCCESS \|\| !std::isfinite(double_val)) {
339	7	return Status::InvalidArgument("invalid number, raw string is: {}", quote_token(token));
340	7	}
341	7	if (!writer.writeDouble(double_val)) {
342	0	return Status::InvalidArgument("writeDouble failed");
343	0	}
344	7	return Status::OK();
345	7	}
346
347		static Status write_number(simdjson::error_code res, simdjson::ondemand::number num,
348	228k	std::string_view raw_string, JsonbWriter& writer) {
349	228k	switch (res) {
350	228k	case simdjson::error_code::SUCCESS:
351	228k	break;
352	52	case simdjson::error_code::NUMBER_ERROR:
353	65	case simdjson::error_code::BIGINT_ERROR:
354	65	return write_number_from_token(res, raw_string, writer);
355	0	default:
356		// simdjson reports no other error for a number token (a root number followed by
357		// another token is already rejected by the end-of-document check in parse()), so
358		// anything else is reported as is.
359	0	return Status::InvalidArgument("simdjson get_number failed: {}, raw string is: {}",
360	0	simdjson::error_message(res), quote_token(raw_string));
361	228k	}
362
363		// On success simdjson yields one of three number types:
364		// 1. floating_point_number: A binary64 number, which will be converted to jsonb's double type.
365		// 2. signed_integer: A signed integer that fits in a 64-bit word using two's complement.
366		// 3. unsigned_integer: A positive integer larger or equal to 1<<63.
367		// For these two integer types, we will convert them to jsonb's int8/int16/int32/int64/int128 types according to the specific value.
368	228k	switch (num.get_number_type()) {
369	115k	case simdjson::ondemand::number_type::floating_point_number: {
370	115k	if (writer.writeDouble(num.get_double()) == 0) {
371	0	return Status::InvalidArgument("writeDouble failed");
372	0	}
373	115k	break;
374	115k	}
375	115k	case simdjson::ondemand::number_type::signed_integer:
376	112k	case simdjson::ondemand::number_type::unsigned_integer: {
377	112k	int128_t val = num.is_int64() ? (int128_t)num.get_int64() : (int128_t)num.get_uint64();
378	112k	bool success = false;
379	112k	if (val >= std::numeric_limits<int8_t>::min() &&
380	112k	val <= std::numeric_limits<int8_t>::max()) {
381	51.5k	success = writer.writeInt8((int8_t)val);
382	61.3k	} else if (val >= std::numeric_limits<int16_t>::min() &&
383	61.3k	val <= std::numeric_limits<int16_t>::max()) {
384	353	success = writer.writeInt16((int16_t)val);
385	61.0k	} else if (val >= std::numeric_limits<int32_t>::min() &&
386	61.0k	val <= std::numeric_limits<int32_t>::max()) {
387	30	success = writer.writeInt32((int32_t)val);
388	60.9k	} else if (val >= std::numeric_limits<int64_t>::min() &&
389	60.9k	val <= std::numeric_limits<int64_t>::max()) {
390	60.9k	success = writer.writeInt64((int64_t)val);
391	60.9k	} else { // INT128
392	13	success = writer.writeInt128(val);
393	13	}
394
395	112k	if (!success) {
396	0	return Status::InvalidArgument("writeInt failed");
397	0	}
398	112k	break;
399	112k	}
400	112k	case simdjson::ondemand::number_type::big_integer: {
401		// simdjson never parses a big_integer successfully; integers beyond 64 bits
402		// arrive as NUMBER_ERROR / BIGINT_ERROR and are handled by
403		// write_number_from_token above.
404	0	__builtin_unreachable();
405	112k	}
406	228k	}
407	228k	return Status::OK();
408	228k	}
409		};
410		} // namespace doris