be/src/exprs/function/cast/cast_to_jsonb.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "core/assert_cast.h" |
19 | | #include "core/data_type/data_type_jsonb.h" |
20 | | #include "core/data_type/data_type_nullable.h" |
21 | | #include "core/data_type/primitive_type.h" |
22 | | #include "core/data_type_serde/data_type_serde.h" |
23 | | #include "core/string_ref.h" |
24 | | #include "core/value/jsonb_value.h" |
25 | | #include "exprs/function/cast/cast_base.h" |
26 | | #include "exprs/function/cast/cast_to_string.h" |
27 | | #include "runtime/runtime_state.h" |
28 | | #include "util/jsonb_utils.h" |
29 | | #include "util/jsonb_writer.h" |
30 | | |
31 | | namespace doris::CastWrapper { |
32 | | |
33 | | struct ConvertImplGenericFromJsonb { |
34 | | static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
35 | | uint32_t result, size_t input_rows_count, |
36 | 5 | const NullMap::value_type* null_map = nullptr) { |
37 | 5 | auto data_type_to = block.get_by_position(result).type; |
38 | 5 | auto data_type_serde_to = data_type_to->get_serde(); |
39 | | |
40 | 5 | DataTypeSerDe::FormatOptions options; |
41 | 5 | options.converted_from_string = true; |
42 | 5 | options.escape_char = '\\'; |
43 | 5 | options.timezone = &context->state()->timezone_obj(); |
44 | | |
45 | 5 | const auto& col_with_type_and_name = block.get_by_position(arguments[0]); |
46 | 5 | const IColumn& col_from = *col_with_type_and_name.column; |
47 | 5 | if (const ColumnString* col_from_string = check_and_get_column<ColumnString>(&col_from)) { |
48 | 5 | auto col_to = data_type_to->create_column(); |
49 | | |
50 | 5 | size_t size = col_from.size(); |
51 | 5 | col_to->reserve(size); |
52 | | |
53 | 5 | ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(size, 0); |
54 | 5 | ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data(); |
55 | 5 | const bool is_complex = is_complex_type(data_type_to->get_primitive_type()); |
56 | 5 | const bool is_dst_string = is_string_type(data_type_to->get_primitive_type()); |
57 | 17 | for (size_t i = 0; i < size; ++i) { |
58 | 12 | if (null_map && null_map[i]) { |
59 | 1 | (*vec_null_map_to)[i] = 1; |
60 | 1 | col_to->insert_default(); |
61 | 1 | continue; |
62 | 1 | } |
63 | 11 | const auto& val = col_from_string->get_data_at(i); |
64 | 11 | const JsonbDocument* doc = nullptr; |
65 | 11 | auto st = JsonbDocument::checkAndCreateDocument(val.data, val.size, &doc); |
66 | 11 | if (!st.ok() || !doc || !doc->getValue()) [[unlikely]] { |
67 | 0 | (*vec_null_map_to)[i] = 1; |
68 | 0 | col_to->insert_default(); |
69 | 0 | continue; |
70 | 0 | } |
71 | | |
72 | | // value is NOT necessary to be deleted since JsonbValue will not allocate memory |
73 | 11 | const JsonbValue* value = doc->getValue(); |
74 | 11 | if (UNLIKELY(!value)) { |
75 | 0 | (*vec_null_map_to)[i] = 1; |
76 | 0 | col_to->insert_default(); |
77 | 0 | continue; |
78 | 0 | } |
79 | | // Note: here we should handle the null element |
80 | 11 | if (val.size == 0) { |
81 | 0 | col_to->insert_default(); |
82 | | // empty string('') is an invalid format for complex type, set null_map to 1 |
83 | 0 | if (is_complex) { |
84 | 0 | (*vec_null_map_to)[i] = 1; |
85 | 0 | } |
86 | 0 | continue; |
87 | 0 | } |
88 | | // add string to string column |
89 | 11 | if (context->jsonb_string_as_string() && is_dst_string && value->isString()) { |
90 | 3 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
91 | 3 | assert_cast<ColumnString&, TypeCheckOnRelease::DISABLE>(*col_to).insert_data( |
92 | 3 | blob->getBlob(), blob->getBlobLen()); |
93 | 3 | (*vec_null_map_to)[i] = 0; |
94 | 3 | continue; |
95 | 3 | } |
96 | 8 | std::string input_str; |
97 | 8 | if (context->jsonb_string_as_string() && value->isString()) { |
98 | 0 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
99 | 0 | input_str = std::string(blob->getBlob(), blob->getBlobLen()); |
100 | 8 | } else { |
101 | 8 | input_str = JsonbToJson::jsonb_to_json_string(val.data, val.size); |
102 | 8 | } |
103 | 8 | if (input_str.empty()) { |
104 | 0 | col_to->insert_default(); |
105 | 0 | (*vec_null_map_to)[i] = 1; |
106 | 0 | continue; |
107 | 0 | } |
108 | 8 | StringRef read_buffer((char*)(input_str.data()), input_str.size()); |
109 | 8 | st = data_type_serde_to->from_string(read_buffer, *col_to, options); |
110 | | // if parsing failed, will return null |
111 | 8 | (*vec_null_map_to)[i] = !st.ok(); |
112 | 8 | if (!st.ok()) { |
113 | 0 | col_to->insert_default(); |
114 | 0 | } |
115 | 8 | } |
116 | 5 | block.get_by_position(result).column = |
117 | 5 | ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); |
118 | 5 | } else { |
119 | 0 | return Status::RuntimeError( |
120 | 0 | "Illegal column {} of first argument of conversion function from string", |
121 | 0 | col_from.get_name()); |
122 | 0 | } |
123 | 5 | return Status::OK(); |
124 | 5 | } |
125 | | }; |
126 | | |
127 | 0 | inline bool can_cast_json_type(PrimitiveType pt) { |
128 | 0 | return is_int_or_bool(pt) || is_float_or_double(pt) || is_string_type(pt) || is_decimal(pt) || |
129 | 0 | pt == TYPE_ARRAY || pt == TYPE_STRUCT; |
130 | 0 | } |
131 | | |
132 | | // check jsonb value type and get to_type value |
133 | | WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type, |
134 | | const DataTypePtr& to_type, |
135 | 133 | bool jsonb_string_as_string) { |
136 | 133 | if (is_string_type(to_type->get_primitive_type()) && jsonb_string_as_string) { |
137 | 5 | return ConvertImplGenericFromJsonb::execute; |
138 | 5 | } |
139 | | |
140 | 128 | return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
141 | 128 | uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { |
142 | 128 | CastParameters params; |
143 | 128 | params.is_strict = context->enable_strict_mode(); |
144 | | |
145 | 128 | auto data_type_to = remove_nullable(block.get_by_position(result).type); |
146 | 128 | auto serde_to = data_type_to->get_serde(); |
147 | | |
148 | 128 | const auto& col_from_json = |
149 | 128 | assert_cast<const ColumnString&>(*block.get_by_position(arguments[0]).column); |
150 | | |
151 | 128 | auto column_to = make_nullable(data_type_to)->create_column(); |
152 | 128 | auto& column_to_nullable = assert_cast<ColumnNullable&>(*column_to); |
153 | | |
154 | 128 | RETURN_IF_ERROR(serde_to->deserialize_column_from_jsonb_vector( |
155 | 128 | column_to_nullable, col_from_json, params, null_map)); |
156 | | |
157 | 125 | block.get_by_position(result).column = std::move(column_to); |
158 | 125 | return Status::OK(); |
159 | 128 | }; |
160 | 133 | } |
161 | | |
162 | | struct ParseJsonbFromString { |
163 | 4 | static Status parse_json(const StringRef& str, ColumnString& column_string) { |
164 | 4 | if (str.empty()) { |
165 | 0 | return Status::InvalidArgument("Empty string cannot be parsed as jsonb"); |
166 | 0 | } |
167 | 4 | JsonBinaryValue value; |
168 | 4 | auto st = (value.from_json_string(str.data, str.size)); |
169 | 4 | if (!st.ok()) { |
170 | 2 | return Status::InvalidArgument("Failed to parse json string: {}, error: {}", |
171 | 2 | str.to_string(), st.msg()); |
172 | 2 | } |
173 | 2 | column_string.insert_data(value.value(), value.size()); |
174 | 2 | return Status::OK(); |
175 | 4 | } |
176 | | |
177 | | static Status execute_non_strict(const ColumnString& col_from, size_t size, |
178 | 4 | ColumnPtr& column_result) { |
179 | 4 | auto col_to = ColumnString::create(); |
180 | 4 | auto col_null = ColumnBool::create(size, 0); |
181 | 4 | auto& vec_null_map_to = col_null->get_data(); |
182 | | |
183 | 8 | for (size_t i = 0; i < size; ++i) { |
184 | 4 | Status st = parse_json(col_from.get_data_at(i), *col_to); |
185 | 4 | vec_null_map_to[i] = !st.ok(); |
186 | 4 | if (!st.ok()) [[unlikely]] { |
187 | 2 | col_to->insert_default(); |
188 | 2 | } |
189 | 4 | } |
190 | 4 | column_result = ColumnNullable::create(std::move(col_to), std::move(col_null)); |
191 | 4 | return Status::OK(); |
192 | 4 | } |
193 | | |
194 | | // in both strict or non-strict mode, the return type is nullable column |
195 | | static Status execute_strict(const ColumnString& col_from, const NullMap::value_type* null_map, |
196 | 0 | size_t size, ColumnPtr& column_result) { |
197 | 0 | auto col_to = ColumnString::create(); |
198 | 0 | for (size_t i = 0; i < size; ++i) { |
199 | 0 | if (null_map && null_map[i]) { |
200 | 0 | col_to->insert_default(); |
201 | 0 | continue; |
202 | 0 | } |
203 | 0 | RETURN_IF_ERROR(parse_json(col_from.get_data_at(i), *col_to)); |
204 | 0 | } |
205 | 0 | column_result = ColumnNullable::create(std::move(col_to), ColumnBool::create(size, 0)); |
206 | 0 | return Status::OK(); |
207 | 0 | } |
208 | | |
209 | | static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
210 | | uint32_t result, size_t input_rows_count, |
211 | 4 | const NullMap::value_type* null_map) { |
212 | 4 | const auto& col_from = |
213 | 4 | assert_cast<const ColumnString&>(*block.get_by_position(arguments[0]).column); |
214 | 4 | const auto size = col_from.size(); |
215 | | |
216 | 4 | ColumnPtr column_result; |
217 | 4 | if (context->enable_strict_mode()) { |
218 | 0 | RETURN_IF_ERROR(execute_strict(col_from, null_map, size, column_result)); |
219 | |
|
220 | 4 | } else { |
221 | 4 | RETURN_IF_ERROR(execute_non_strict(col_from, size, column_result)); |
222 | 4 | } |
223 | 4 | block.get_by_position(result).column = std::move(column_result); |
224 | | |
225 | 4 | return Status::OK(); |
226 | 4 | } |
227 | | }; |
228 | | |
229 | | // create corresponding jsonb value with type to_type |
230 | | // use jsonb writer to create jsonb value |
231 | | WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const DataTypeJsonb& to_type, |
232 | 66 | bool string_as_jsonb_string) { |
233 | | // parse string as jsonb |
234 | 66 | if (is_string_type(from_type->get_primitive_type()) && !string_as_jsonb_string) { |
235 | 4 | return ParseJsonbFromString::execute; |
236 | 4 | } |
237 | | |
238 | 62 | return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
239 | 62 | uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { |
240 | | // same as to_json function |
241 | 62 | auto to_column = ColumnString::create(); |
242 | 62 | auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); |
243 | 62 | auto from_column = block.get_by_position(arguments[0]).column; |
244 | 62 | DataTypeSerDe::FormatOptions options; |
245 | 62 | options.timezone = &context->state()->timezone_obj(); |
246 | 62 | RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, |
247 | 62 | options, null_map)); |
248 | 48 | block.get_by_position(result).column = std::move(to_column); |
249 | 48 | return Status::OK(); |
250 | 62 | }; |
251 | 66 | } |
252 | | } // namespace doris::CastWrapper |