be/src/exprs/function/cast/cast_to_jsonb.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "core/assert_cast.h" |
19 | | #include "core/data_type/data_type_jsonb.h" |
20 | | #include "core/data_type/data_type_nullable.h" |
21 | | #include "core/data_type/primitive_type.h" |
22 | | #include "core/data_type_serde/data_type_serde.h" |
23 | | #include "core/string_ref.h" |
24 | | #include "core/value/jsonb_value.h" |
25 | | #include "exprs/function/cast/cast_base.h" |
26 | | #include "exprs/function/cast/cast_to_string.h" |
27 | | #include "util/jsonb_utils.h" |
28 | | #include "util/jsonb_writer.h" |
29 | | |
30 | | namespace doris::CastWrapper { |
31 | | |
32 | | struct ConvertImplGenericFromJsonb { |
33 | | static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
34 | | uint32_t result, size_t input_rows_count, |
35 | 5 | const NullMap::value_type* null_map = nullptr) { |
36 | 5 | auto data_type_to = block.get_by_position(result).type; |
37 | 5 | auto data_type_serde_to = data_type_to->get_serde(); |
38 | | |
39 | 5 | DataTypeSerDe::FormatOptions options; |
40 | 5 | options.converted_from_string = true; |
41 | 5 | options.escape_char = '\\'; |
42 | 5 | options.timezone = &context->state()->timezone_obj(); |
43 | | |
44 | 5 | const auto& col_with_type_and_name = block.get_by_position(arguments[0]); |
45 | 5 | const IColumn& col_from = *col_with_type_and_name.column; |
46 | 5 | if (const ColumnString* col_from_string = check_and_get_column<ColumnString>(&col_from)) { |
47 | 5 | auto col_to = data_type_to->create_column(); |
48 | | |
49 | 5 | size_t size = col_from.size(); |
50 | 5 | col_to->reserve(size); |
51 | | |
52 | 5 | ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(size, 0); |
53 | 5 | ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data(); |
54 | 5 | const bool is_complex = is_complex_type(data_type_to->get_primitive_type()); |
55 | 5 | const bool is_dst_string = is_string_type(data_type_to->get_primitive_type()); |
56 | 17 | for (size_t i = 0; i < size; ++i) { |
57 | 12 | if (null_map && null_map[i]) { |
58 | 1 | (*vec_null_map_to)[i] = 1; |
59 | 1 | col_to->insert_default(); |
60 | 1 | continue; |
61 | 1 | } |
62 | 11 | const auto& val = col_from_string->get_data_at(i); |
63 | 11 | const JsonbDocument* doc = nullptr; |
64 | 11 | auto st = JsonbDocument::checkAndCreateDocument(val.data, val.size, &doc); |
65 | 11 | if (!st.ok() || !doc || !doc->getValue()) [[unlikely]] { |
66 | 0 | (*vec_null_map_to)[i] = 1; |
67 | 0 | col_to->insert_default(); |
68 | 0 | continue; |
69 | 0 | } |
70 | | |
71 | | // value is NOT necessary to be deleted since JsonbValue will not allocate memory |
72 | 11 | const JsonbValue* value = doc->getValue(); |
73 | 11 | if (UNLIKELY(!value)) { |
74 | 0 | (*vec_null_map_to)[i] = 1; |
75 | 0 | col_to->insert_default(); |
76 | 0 | continue; |
77 | 0 | } |
78 | | // Note: here we should handle the null element |
79 | 11 | if (val.size == 0) { |
80 | 0 | col_to->insert_default(); |
81 | | // empty string('') is an invalid format for complex type, set null_map to 1 |
82 | 0 | if (is_complex) { |
83 | 0 | (*vec_null_map_to)[i] = 1; |
84 | 0 | } |
85 | 0 | continue; |
86 | 0 | } |
87 | | // add string to string column |
88 | 11 | if (context->jsonb_string_as_string() && is_dst_string && value->isString()) { |
89 | 3 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
90 | 3 | assert_cast<ColumnString&, TypeCheckOnRelease::DISABLE>(*col_to).insert_data( |
91 | 3 | blob->getBlob(), blob->getBlobLen()); |
92 | 3 | (*vec_null_map_to)[i] = 0; |
93 | 3 | continue; |
94 | 3 | } |
95 | 8 | std::string input_str; |
96 | 8 | if (context->jsonb_string_as_string() && value->isString()) { |
97 | 0 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
98 | 0 | input_str = std::string(blob->getBlob(), blob->getBlobLen()); |
99 | 8 | } else { |
100 | 8 | input_str = JsonbToJson::jsonb_to_json_string(val.data, val.size); |
101 | 8 | } |
102 | 8 | if (input_str.empty()) { |
103 | 0 | col_to->insert_default(); |
104 | 0 | (*vec_null_map_to)[i] = 1; |
105 | 0 | continue; |
106 | 0 | } |
107 | 8 | StringRef read_buffer((char*)(input_str.data()), input_str.size()); |
108 | 8 | st = data_type_serde_to->from_string(read_buffer, *col_to, options); |
109 | | // if parsing failed, will return null |
110 | 8 | (*vec_null_map_to)[i] = !st.ok(); |
111 | 8 | if (!st.ok()) { |
112 | 0 | col_to->insert_default(); |
113 | 0 | } |
114 | 8 | } |
115 | 5 | block.get_by_position(result).column = |
116 | 5 | ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); |
117 | 5 | } else { |
118 | 0 | return Status::RuntimeError( |
119 | 0 | "Illegal column {} of first argument of conversion function from string", |
120 | 0 | col_from.get_name()); |
121 | 0 | } |
122 | 5 | return Status::OK(); |
123 | 5 | } |
124 | | }; |
125 | | |
126 | 0 | inline bool can_cast_json_type(PrimitiveType pt) { |
127 | 0 | return is_int_or_bool(pt) || is_float_or_double(pt) || is_string_type(pt) || is_decimal(pt) || |
128 | 0 | pt == TYPE_ARRAY || pt == TYPE_STRUCT; |
129 | 0 | } |
130 | | |
131 | | // check jsonb value type and get to_type value |
132 | | WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type, |
133 | | const DataTypePtr& to_type, |
134 | 133 | bool jsonb_string_as_string) { |
135 | 133 | if (is_string_type(to_type->get_primitive_type()) && jsonb_string_as_string) { |
136 | 5 | return ConvertImplGenericFromJsonb::execute; |
137 | 5 | } |
138 | | |
139 | 128 | return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
140 | 128 | uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { |
141 | 128 | CastParameters params; |
142 | 128 | params.is_strict = context->enable_strict_mode(); |
143 | | |
144 | 128 | auto data_type_to = remove_nullable(block.get_by_position(result).type); |
145 | 128 | auto serde_to = data_type_to->get_serde(); |
146 | | |
147 | 128 | const auto& col_from_json = |
148 | 128 | assert_cast<const ColumnString&>(*block.get_by_position(arguments[0]).column); |
149 | | |
150 | 128 | auto column_to = make_nullable(data_type_to)->create_column(); |
151 | 128 | auto& column_to_nullable = assert_cast<ColumnNullable&>(*column_to); |
152 | | |
153 | 128 | RETURN_IF_ERROR(serde_to->deserialize_column_from_jsonb_vector( |
154 | 128 | column_to_nullable, col_from_json, params, null_map)); |
155 | | |
156 | 125 | block.get_by_position(result).column = std::move(column_to); |
157 | 125 | return Status::OK(); |
158 | 128 | }; |
159 | 133 | } |
160 | | |
161 | | struct ParseJsonbFromString { |
162 | 4 | static Status parse_json(const StringRef& str, ColumnString& column_string) { |
163 | 4 | if (str.empty()) { |
164 | 0 | return Status::InvalidArgument("Empty string cannot be parsed as jsonb"); |
165 | 0 | } |
166 | 4 | JsonBinaryValue value; |
167 | 4 | auto st = (value.from_json_string(str.data, str.size)); |
168 | 4 | if (!st.ok()) { |
169 | 2 | return Status::InvalidArgument("Failed to parse json string: {}, error: {}", |
170 | 2 | str.to_string(), st.msg()); |
171 | 2 | } |
172 | 2 | column_string.insert_data(value.value(), value.size()); |
173 | 2 | return Status::OK(); |
174 | 4 | } |
175 | | |
176 | | static Status execute_non_strict(const ColumnString& col_from, size_t size, |
177 | 4 | ColumnPtr& column_result) { |
178 | 4 | auto col_to = ColumnString::create(); |
179 | 4 | auto col_null = ColumnBool::create(size, 0); |
180 | 4 | auto& vec_null_map_to = col_null->get_data(); |
181 | | |
182 | 8 | for (size_t i = 0; i < size; ++i) { |
183 | 4 | Status st = parse_json(col_from.get_data_at(i), *col_to); |
184 | 4 | vec_null_map_to[i] = !st.ok(); |
185 | 4 | if (!st.ok()) [[unlikely]] { |
186 | 2 | col_to->insert_default(); |
187 | 2 | } |
188 | 4 | } |
189 | 4 | column_result = ColumnNullable::create(std::move(col_to), std::move(col_null)); |
190 | 4 | return Status::OK(); |
191 | 4 | } |
192 | | |
193 | | // in both strict or non-strict mode, the return type is nullable column |
194 | | static Status execute_strict(const ColumnString& col_from, const NullMap::value_type* null_map, |
195 | 0 | size_t size, ColumnPtr& column_result) { |
196 | 0 | auto col_to = ColumnString::create(); |
197 | 0 | for (size_t i = 0; i < size; ++i) { |
198 | 0 | if (null_map && null_map[i]) { |
199 | 0 | col_to->insert_default(); |
200 | 0 | continue; |
201 | 0 | } |
202 | 0 | RETURN_IF_ERROR(parse_json(col_from.get_data_at(i), *col_to)); |
203 | 0 | } |
204 | 0 | column_result = ColumnNullable::create(std::move(col_to), ColumnBool::create(size, 0)); |
205 | 0 | return Status::OK(); |
206 | 0 | } |
207 | | |
208 | | static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
209 | | uint32_t result, size_t input_rows_count, |
210 | 4 | const NullMap::value_type* null_map) { |
211 | 4 | const auto& col_from = |
212 | 4 | assert_cast<const ColumnString&>(*block.get_by_position(arguments[0]).column); |
213 | 4 | const auto size = col_from.size(); |
214 | | |
215 | 4 | ColumnPtr column_result; |
216 | 4 | if (context->enable_strict_mode()) { |
217 | 0 | RETURN_IF_ERROR(execute_strict(col_from, null_map, size, column_result)); |
218 | |
|
219 | 4 | } else { |
220 | 4 | RETURN_IF_ERROR(execute_non_strict(col_from, size, column_result)); |
221 | 4 | } |
222 | 4 | block.get_by_position(result).column = std::move(column_result); |
223 | | |
224 | 4 | return Status::OK(); |
225 | 4 | } |
226 | | }; |
227 | | |
228 | | // create corresponding jsonb value with type to_type |
229 | | // use jsonb writer to create jsonb value |
230 | | WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const DataTypeJsonb& to_type, |
231 | 66 | bool string_as_jsonb_string) { |
232 | | // parse string as jsonb |
233 | 66 | if (is_string_type(from_type->get_primitive_type()) && !string_as_jsonb_string) { |
234 | 4 | return ParseJsonbFromString::execute; |
235 | 4 | } |
236 | | |
237 | 62 | return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
238 | 62 | uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { |
239 | | // same as to_json function |
240 | 62 | auto to_column = ColumnString::create(); |
241 | 62 | auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); |
242 | 62 | auto from_column = block.get_by_position(arguments[0]).column; |
243 | 62 | RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, |
244 | 62 | null_map)); |
245 | 48 | block.get_by_position(result).column = std::move(to_column); |
246 | 48 | return Status::OK(); |
247 | 62 | }; |
248 | 66 | } |
249 | | } // namespace doris::CastWrapper |