be/src/format_v2/jni/jdbc_reader.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "format_v2/jni/jdbc_reader.h" |
19 | | |
20 | | #include <memory> |
21 | | #include <utility> |
22 | | |
23 | | #include "common/cast_set.h" |
24 | | #include "core/assert_cast.h" |
25 | | #include "core/block/block.h" |
26 | | #include "core/block/columns_with_type_and_name.h" |
27 | | #include "core/column/column_nullable.h" |
28 | | #include "core/data_type/data_type_nullable.h" |
29 | | #include "core/data_type/data_type_string.h" |
30 | | #include "exprs/function/simple_function_factory.h" |
31 | | #include "exprs/vexpr_context.h" |
32 | | #include "format_v2/table_reader.h" |
33 | | #include "util/jdbc_utils.h" |
34 | | |
35 | | namespace doris::format::jdbc { |
36 | | |
37 | 2 | Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows) { |
38 | 2 | const auto* nullable = check_and_get_column<ColumnNullable>(&result); |
39 | 2 | if (UNLIKELY(nullable == nullptr)) { |
40 | 0 | return Status::InternalError("JDBC special-type CAST did not return a nullable column"); |
41 | 0 | } |
42 | 2 | if (UNLIKELY(nullable->has_null(0, rows))) { |
43 | | // CAST NULL represents invalid source data; stripping the null map would turn it into a |
44 | | // valid-looking default for a NOT NULL destination. |
45 | 1 | return Status::DataQualityError( |
46 | 1 | "JDBC special-type CAST produced NULL for a non-nullable column"); |
47 | 1 | } |
48 | 1 | return Status::OK(); |
49 | 2 | } |
50 | | |
51 | 0 | std::string JdbcJniReader::connector_class() const { |
52 | 0 | return "org/apache/doris/jdbc/JdbcJniScanner"; |
53 | 0 | } |
54 | | |
55 | 0 | Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { |
56 | 0 | { |
57 | | // End these scopes before JniTableReader enters the same counters; nested use would count |
58 | | // this JDBC parameter preparation twice instead of extending the common lifecycle total. |
59 | 0 | SCOPED_TIMER(_profile.total_timer); |
60 | 0 | SCOPED_TIMER(_profile.prepare_split_timer); |
61 | 0 | SCOPED_TIMER(connector_total_timer()); |
62 | 0 | _jdbc_params.clear(); |
63 | 0 | if (options.current_range.__isset.table_format_params && |
64 | 0 | options.current_range.table_format_params.table_format_type == "jdbc") { |
65 | 0 | _jdbc_params = std::map<std::string, std::string>( |
66 | 0 | options.current_range.table_format_params.jdbc_params.begin(), |
67 | 0 | options.current_range.table_format_params.jdbc_params.end()); |
68 | 0 | } |
69 | 0 | } |
70 | 0 | return format::JniTableReader::prepare_split(options); |
71 | 0 | } |
72 | | |
73 | | // need pass to the java side, so the java scanner can parse the params and construct the JDBC connection |
74 | 0 | Status JdbcJniReader::build_scanner_params(std::map<std::string, std::string>* params) const { |
75 | 0 | DORIS_CHECK(params != nullptr); |
76 | 0 | *params = _jdbc_params; |
77 | 0 | if (params->contains("jdbc_driver_url")) { |
78 | 0 | std::string resolved; |
79 | 0 | if (JdbcUtils::resolve_driver_url((*params)["jdbc_driver_url"], &resolved).ok()) { |
80 | 0 | (*params)["jdbc_driver_url"] = resolved; |
81 | 0 | } |
82 | 0 | } |
83 | 0 | return Status::OK(); |
84 | 0 | } |
85 | | |
86 | | Status JdbcJniReader::build_jni_columns( |
87 | 0 | std::vector<format::JniTableReader::JniColumn>* columns) const { |
88 | 0 | DORIS_CHECK(columns != nullptr); |
89 | 0 | columns->clear(); |
90 | 0 | columns->reserve(_projected_columns.size()); |
91 | 0 | for (size_t i = 0; i < _projected_columns.size(); ++i) { |
92 | 0 | const auto& table_column = _projected_columns[i]; |
93 | 0 | const auto primitive_type = remove_nullable(table_column.type)->get_primitive_type(); |
94 | 0 | columns->push_back({ |
95 | 0 | .java_name = table_column.name, |
96 | 0 | .output_index = i, |
97 | 0 | .output_type = table_column.type, |
98 | 0 | .transfer_type = _transfer_type_for(table_column.type), |
99 | 0 | .replace_type = _replace_type_for(primitive_type), |
100 | 0 | }); |
101 | 0 | } |
102 | 0 | return Status::OK(); |
103 | 0 | } |
104 | | |
105 | 0 | Status JdbcJniReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { |
106 | 0 | DORIS_CHECK(jni_block != nullptr); |
107 | 0 | DORIS_CHECK(output_block != nullptr); |
108 | 0 | DORIS_CHECK(rows != nullptr); |
109 | 0 | const auto original_rows = *rows; |
110 | 0 | const auto& columns = jni_columns(); |
111 | 0 | DORIS_CHECK(columns.size() == jni_block->columns()); |
112 | |
|
113 | 0 | for (size_t i = 0; i < columns.size(); ++i) { |
114 | 0 | const auto& column = columns[i]; |
115 | 0 | DORIS_CHECK(column.output_type != nullptr); |
116 | 0 | DORIS_CHECK(column.output_index < output_block->columns()); |
117 | 0 | if (_is_special_type(remove_nullable(column.output_type)->get_primitive_type())) { |
118 | 0 | RETURN_IF_ERROR(_cast_string_to_special_type(column, jni_block, i, output_block, |
119 | 0 | original_rows)); |
120 | 0 | continue; |
121 | 0 | } |
122 | 0 | output_block->get_by_position(column.output_index).type = column.output_type; |
123 | 0 | output_block->replace_by_position(column.output_index, |
124 | 0 | jni_block->get_by_position(i).column); |
125 | 0 | } |
126 | 0 | DORIS_CHECK(output_block->rows() == original_rows); |
127 | 0 | if (!_conjuncts.empty()) { |
128 | 0 | RETURN_IF_ERROR( |
129 | 0 | VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); |
130 | 0 | } |
131 | 0 | *rows = output_block->rows(); |
132 | 0 | return Status::OK(); |
133 | 0 | } |
134 | | |
135 | 0 | std::string JdbcJniReader::_replace_type_for(PrimitiveType type) const { |
136 | 0 | switch (type) { |
137 | 0 | case PrimitiveType::TYPE_BITMAP: |
138 | 0 | return "bitmap"; |
139 | 0 | case PrimitiveType::TYPE_HLL: |
140 | 0 | return "hll"; |
141 | 0 | case PrimitiveType::TYPE_QUANTILE_STATE: |
142 | 0 | return "quantile_state"; |
143 | 0 | case PrimitiveType::TYPE_JSONB: |
144 | 0 | return "jsonb"; |
145 | 0 | default: |
146 | 0 | return "not_replace"; |
147 | 0 | } |
148 | 0 | } |
149 | | |
150 | 0 | bool JdbcJniReader::_is_special_type(PrimitiveType type) const { |
151 | 0 | return type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_HLL || |
152 | 0 | type == PrimitiveType::TYPE_QUANTILE_STATE || type == PrimitiveType::TYPE_JSONB; |
153 | 0 | } |
154 | | |
155 | 0 | DataTypePtr JdbcJniReader::_transfer_type_for(const DataTypePtr& output_type) const { |
156 | 0 | DORIS_CHECK(output_type != nullptr); |
157 | 0 | if (!_is_special_type(remove_nullable(output_type)->get_primitive_type())) { |
158 | 0 | return output_type; |
159 | 0 | } |
160 | 0 | DataTypePtr string_type = std::make_shared<DataTypeString>(); |
161 | 0 | if (output_type->is_nullable()) { |
162 | 0 | string_type = make_nullable(string_type); |
163 | 0 | } |
164 | 0 | return string_type; |
165 | 0 | } |
166 | | |
167 | | Status JdbcJniReader::_cast_string_to_special_type(const format::JniTableReader::JniColumn& column, |
168 | | Block* jni_block, size_t jni_column_index, |
169 | 0 | Block* output_block, size_t rows) { |
170 | 0 | DORIS_CHECK(column.output_type != nullptr); |
171 | 0 | DORIS_CHECK(column.transfer_type != nullptr); |
172 | 0 | const auto target_type = column.output_type; |
173 | 0 | const auto target_type_name = target_type->get_name(); |
174 | |
|
175 | 0 | ColumnPtr input_column = jni_block->get_by_position(jni_column_index).column; |
176 | 0 | ColumnPtr cast_param = target_type->create_column_const_with_default_value(1); |
177 | |
|
178 | 0 | ColumnsWithTypeAndName argument_template; |
179 | 0 | argument_template.reserve(2); |
180 | 0 | argument_template.emplace_back(std::move(input_column), column.transfer_type, |
181 | 0 | "java.sql.String"); |
182 | 0 | argument_template.emplace_back(std::move(cast_param), target_type, target_type_name); |
183 | |
|
184 | 0 | FunctionBasePtr cast_function = SimpleFunctionFactory::instance().get_function( |
185 | 0 | "CAST", argument_template, make_nullable(target_type)); |
186 | 0 | if (cast_function == nullptr) { |
187 | 0 | return Status::InternalError("Failed to find CAST function for type {}", target_type_name); |
188 | 0 | } |
189 | | |
190 | 0 | Block cast_block(argument_template); |
191 | 0 | const auto result_idx = cast_set<uint32_t>(cast_block.columns()); |
192 | 0 | cast_block.insert({nullptr, make_nullable(target_type), "cast_result"}); |
193 | 0 | RETURN_IF_ERROR( |
194 | 0 | cast_function->execute(nullptr, cast_block, {0}, result_idx, cast_set<int>(rows))); |
195 | | |
196 | 0 | auto result_column = cast_block.get_by_position(result_idx).column; |
197 | 0 | output_block->get_by_position(column.output_index).type = target_type; |
198 | 0 | if (target_type->is_nullable()) { |
199 | 0 | output_block->replace_by_position(column.output_index, result_column); |
200 | 0 | } else { |
201 | 0 | RETURN_IF_ERROR(validate_non_nullable_special_type_result(*result_column, rows)); |
202 | 0 | const auto* nullable_column = assert_cast<const ColumnNullable*>(result_column.get()); |
203 | 0 | output_block->replace_by_position(column.output_index, |
204 | 0 | nullable_column->get_nested_column_ptr()); |
205 | 0 | } |
206 | 0 | return Status::OK(); |
207 | 0 | } |
208 | | |
209 | | } // namespace doris::format::jdbc |