be/src/exprs/function/cast/cast_to_jsonb.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "core/assert_cast.h" |
19 | | #include "core/data_type/data_type_jsonb.h" |
20 | | #include "core/data_type/data_type_nullable.h" |
21 | | #include "core/data_type/primitive_type.h" |
22 | | #include "core/data_type_serde/data_type_serde.h" |
23 | | #include "core/string_ref.h" |
24 | | #include "core/value/jsonb_value.h" |
25 | | #include "exprs/function/cast/cast_base.h" |
26 | | #include "exprs/function/cast/cast_to_string.h" |
27 | | #include "runtime/runtime_state.h" |
28 | | #include "util/jsonb_utils.h" |
29 | | #include "util/jsonb_writer.h" |
30 | | |
31 | | namespace doris::CastWrapper { |
32 | | |
33 | | struct ConvertImplGenericFromJsonb { |
34 | | static Status execute(RuntimeState* state, const ColumnWithTypeAndName& source_column, |
35 | | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
36 | 5 | const NullMap::value_type* null_map, bool jsonb_string_as_string) { |
37 | 5 | auto data_type_to = target_column.type; |
38 | 5 | auto data_type_serde_to = data_type_to->get_serde(); |
39 | | |
40 | 5 | DataTypeSerDe::FormatOptions options; |
41 | 5 | options.converted_from_string = true; |
42 | 5 | options.escape_char = '\\'; |
43 | 5 | options.timezone = &state->timezone_obj(); |
44 | | |
45 | 5 | const IColumn& col_from = *source_column.column; |
46 | 5 | if (const ColumnString* col_from_string = check_and_get_column<ColumnString>(&col_from)) { |
47 | 5 | auto col_to = data_type_to->create_column(); |
48 | | |
49 | 5 | size_t size = col_from.size(); |
50 | 5 | col_to->reserve(size); |
51 | | |
52 | 5 | ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(size, 0); |
53 | 5 | ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data(); |
54 | 5 | const bool is_complex = is_complex_type(data_type_to->get_primitive_type()); |
55 | 5 | const bool is_dst_string = is_string_type(data_type_to->get_primitive_type()); |
56 | 17 | for (size_t i = 0; i < size; ++i) { |
57 | 12 | if (null_map && null_map[i]) { |
58 | 1 | (*vec_null_map_to)[i] = 1; |
59 | 1 | col_to->insert_default(); |
60 | 1 | continue; |
61 | 1 | } |
62 | 11 | const auto& val = col_from_string->get_data_at(i); |
63 | 11 | const JsonbDocument* doc = nullptr; |
64 | 11 | auto st = JsonbDocument::checkAndCreateDocument(val.data, val.size, &doc); |
65 | 11 | if (!st.ok() || !doc || !doc->getValue()) [[unlikely]] { |
66 | 0 | (*vec_null_map_to)[i] = 1; |
67 | 0 | col_to->insert_default(); |
68 | 0 | continue; |
69 | 0 | } |
70 | | |
71 | | // value is NOT necessary to be deleted since JsonbValue will not allocate memory |
72 | 11 | const JsonbValue* value = doc->getValue(); |
73 | 11 | if (UNLIKELY(!value)) { |
74 | 0 | (*vec_null_map_to)[i] = 1; |
75 | 0 | col_to->insert_default(); |
76 | 0 | continue; |
77 | 0 | } |
78 | | // Note: here we should handle the null element |
79 | 11 | if (val.size == 0) { |
80 | 0 | col_to->insert_default(); |
81 | | // empty string('') is an invalid format for complex type, set null_map to 1 |
82 | 0 | if (is_complex) { |
83 | 0 | (*vec_null_map_to)[i] = 1; |
84 | 0 | } |
85 | 0 | continue; |
86 | 0 | } |
87 | | // add string to string column |
88 | 11 | if (jsonb_string_as_string && is_dst_string && value->isString()) { |
89 | 3 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
90 | 3 | assert_cast<ColumnString&, TypeCheckOnRelease::DISABLE>(*col_to).insert_data( |
91 | 3 | blob->getBlob(), blob->getBlobLen()); |
92 | 3 | (*vec_null_map_to)[i] = 0; |
93 | 3 | continue; |
94 | 3 | } |
95 | 8 | std::string input_str; |
96 | 8 | if (jsonb_string_as_string && value->isString()) { |
97 | 0 | const auto* blob = value->unpack<JsonbBinaryVal>(); |
98 | 0 | input_str = std::string(blob->getBlob(), blob->getBlobLen()); |
99 | 8 | } else { |
100 | 8 | input_str = JsonbToJson::jsonb_to_json_string(val.data, val.size); |
101 | 8 | } |
102 | 8 | if (input_str.empty()) { |
103 | 0 | col_to->insert_default(); |
104 | 0 | (*vec_null_map_to)[i] = 1; |
105 | 0 | continue; |
106 | 0 | } |
107 | 8 | StringRef read_buffer((char*)(input_str.data()), input_str.size()); |
108 | 8 | st = data_type_serde_to->from_string(read_buffer, *col_to, options); |
109 | | // if parsing failed, will return null |
110 | 8 | (*vec_null_map_to)[i] = !st.ok(); |
111 | 8 | if (!st.ok()) { |
112 | 0 | col_to->insert_default(); |
113 | 0 | } |
114 | 8 | } |
115 | 5 | target_column.column = |
116 | 5 | ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); |
117 | 5 | } else { |
118 | 0 | return Status::RuntimeError( |
119 | 0 | "Illegal column {} of first argument of conversion function from string", |
120 | 0 | col_from.get_name()); |
121 | 0 | } |
122 | 5 | return Status::OK(); |
123 | 5 | } |
124 | | }; |
125 | | |
126 | 0 | inline bool can_cast_json_type(PrimitiveType pt) { |
127 | 0 | return is_int_or_bool(pt) || is_float_or_double(pt) || is_string_type(pt) || is_decimal(pt) || |
128 | 0 | pt == TYPE_ARRAY || pt == TYPE_STRUCT; |
129 | 0 | } |
130 | | |
131 | | // check jsonb value type and get to_type value |
132 | | WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type, |
133 | 127 | const DataTypePtr& to_type, const CastOptions& options) { |
134 | 127 | if (is_string_type(to_type->get_primitive_type()) && options.jsonb_string_as_string) { |
135 | 5 | return [jsonb_string_as_string = options.jsonb_string_as_string]( |
136 | 5 | RuntimeState* state, const ColumnWithTypeAndName& source_column, |
137 | 5 | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
138 | 5 | const NullMap::value_type* null_map) { |
139 | 5 | return ConvertImplGenericFromJsonb::execute(state, source_column, target_column, |
140 | 5 | input_rows_count, null_map, |
141 | 5 | jsonb_string_as_string); |
142 | 5 | }; |
143 | 5 | } |
144 | | |
145 | 122 | return [strict_mode = options.strict_mode]( |
146 | 122 | RuntimeState* /*state*/, const ColumnWithTypeAndName& source_column, |
147 | 122 | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
148 | 128 | const NullMap::value_type* parent_null_map) { |
149 | 128 | CastParameters params; |
150 | 128 | params.is_strict = strict_mode; |
151 | | |
152 | 128 | auto data_type_to = remove_nullable(target_column.type); |
153 | 128 | auto serde_to = data_type_to->get_serde(); |
154 | | |
155 | 128 | const auto& col_from_json = assert_cast<const ColumnString&>(*source_column.column); |
156 | | |
157 | 128 | auto column_to = make_nullable(data_type_to)->create_column(); |
158 | 128 | auto& column_to_nullable = assert_cast<ColumnNullable&>(*column_to); |
159 | | |
160 | 128 | RETURN_IF_ERROR(serde_to->deserialize_column_from_jsonb_vector( |
161 | 128 | column_to_nullable, col_from_json, params, parent_null_map)); |
162 | | |
163 | 125 | target_column.column = std::move(column_to); |
164 | 125 | return Status::OK(); |
165 | 128 | }; |
166 | 127 | } |
167 | | |
168 | | struct ParseJsonbFromString { |
169 | 4 | static Status parse_json(const StringRef& str, ColumnString& column_string) { |
170 | 4 | if (str.empty()) { |
171 | 0 | return Status::InvalidArgument("Empty string cannot be parsed as jsonb"); |
172 | 0 | } |
173 | 4 | JsonBinaryValue value; |
174 | 4 | auto st = (value.from_json_string(str.data, str.size)); |
175 | 4 | if (!st.ok()) { |
176 | 2 | return Status::InvalidArgument("Failed to parse json string: {}, error: {}", |
177 | 2 | str.to_string(), st.msg()); |
178 | 2 | } |
179 | 2 | column_string.insert_data(value.value(), value.size()); |
180 | 2 | return Status::OK(); |
181 | 4 | } |
182 | | |
183 | | static Status execute_non_strict(const ColumnString& col_from, size_t size, |
184 | 4 | ColumnPtr& column_result) { |
185 | 4 | auto col_to = ColumnString::create(); |
186 | 4 | auto col_null = ColumnBool::create(size, 0); |
187 | 4 | auto& vec_null_map_to = col_null->get_data(); |
188 | | |
189 | 8 | for (size_t i = 0; i < size; ++i) { |
190 | 4 | Status st = parse_json(col_from.get_data_at(i), *col_to); |
191 | 4 | vec_null_map_to[i] = !st.ok(); |
192 | 4 | if (!st.ok()) [[unlikely]] { |
193 | 2 | col_to->insert_default(); |
194 | 2 | } |
195 | 4 | } |
196 | 4 | column_result = ColumnNullable::create(std::move(col_to), std::move(col_null)); |
197 | 4 | return Status::OK(); |
198 | 4 | } |
199 | | |
200 | | // in both strict or non-strict mode, the return type is nullable column |
201 | | static Status execute_strict(const ColumnString& col_from, const NullMap::value_type* null_map, |
202 | 0 | size_t size, ColumnPtr& column_result) { |
203 | 0 | auto col_to = ColumnString::create(); |
204 | 0 | for (size_t i = 0; i < size; ++i) { |
205 | 0 | if (null_map && null_map[i]) { |
206 | 0 | col_to->insert_default(); |
207 | 0 | continue; |
208 | 0 | } |
209 | 0 | RETURN_IF_ERROR(parse_json(col_from.get_data_at(i), *col_to)); |
210 | 0 | } |
211 | 0 | column_result = ColumnNullable::create(std::move(col_to), ColumnBool::create(size, 0)); |
212 | 0 | return Status::OK(); |
213 | 0 | } |
214 | | |
215 | | static Status execute(const ColumnWithTypeAndName& source_column, |
216 | | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
217 | 4 | const NullMap::value_type* null_map, bool strict_mode) { |
218 | 4 | const auto& col_from = assert_cast<const ColumnString&>(*source_column.column); |
219 | 4 | const auto size = col_from.size(); |
220 | | |
221 | 4 | ColumnPtr column_result; |
222 | 4 | if (strict_mode) { |
223 | 0 | RETURN_IF_ERROR(execute_strict(col_from, null_map, size, column_result)); |
224 | |
|
225 | 4 | } else { |
226 | 4 | RETURN_IF_ERROR(execute_non_strict(col_from, size, column_result)); |
227 | 4 | } |
228 | 4 | target_column.column = std::move(column_result); |
229 | | |
230 | 4 | return Status::OK(); |
231 | 4 | } |
232 | | }; |
233 | | |
234 | | // create corresponding jsonb value with type to_type |
235 | | // use jsonb writer to create jsonb value |
236 | | WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const DataTypeJsonb& to_type, |
237 | 71 | const CastOptions& options) { |
238 | | // parse string as jsonb |
239 | 71 | if (is_string_type(from_type->get_primitive_type()) && !options.string_as_jsonb_string) { |
240 | 4 | return [strict_mode = options.strict_mode]( |
241 | 4 | RuntimeState* /*state*/, const ColumnWithTypeAndName& source_column, |
242 | 4 | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
243 | 4 | const NullMap::value_type* null_map) { |
244 | 4 | return ParseJsonbFromString::execute(source_column, target_column, input_rows_count, |
245 | 4 | null_map, strict_mode); |
246 | 4 | }; |
247 | 4 | } |
248 | | |
249 | 67 | return [](RuntimeState* state, const ColumnWithTypeAndName& source_column, |
250 | 67 | ColumnWithTypeAndName& target_column, size_t input_rows_count, |
251 | 85 | const NullMap::value_type* null_map) { |
252 | | // same as to_json function |
253 | 85 | auto to_column = ColumnString::create(); |
254 | 85 | auto from_type_serde = source_column.type->get_serde(); |
255 | 85 | auto from_column = source_column.column; |
256 | 85 | DataTypeSerDe::FormatOptions options; |
257 | 85 | options.timezone = &state->timezone_obj(); |
258 | 85 | RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, |
259 | 85 | options, null_map)); |
260 | 71 | target_column.column = std::move(to_column); |
261 | 71 | return Status::OK(); |
262 | 85 | }; |
263 | 71 | } |
264 | | } // namespace doris::CastWrapper |