/root/doris/be/src/vec/utils/util.hpp
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <thrift/protocol/TJSONProtocol.h> |
21 | | |
22 | | #include <boost/shared_ptr.hpp> |
23 | | |
24 | | #include "runtime/descriptors.h" |
25 | | #include "util/simd/bits.h" |
26 | | #include "vec/columns/column.h" |
27 | | #include "vec/columns/column_nullable.h" |
28 | | #include "vec/core/block.h" |
29 | | #include "vec/exprs/vexpr.h" |
30 | | #include "vec/exprs/vexpr_context.h" |
31 | | |
32 | | namespace doris::vectorized { |
33 | | class VectorizedUtils { |
34 | | public: |
35 | 0 | static Block create_empty_columnswithtypename(const RowDescriptor& row_desc) { |
36 | | // Block block; |
37 | 0 | return create_columns_with_type_and_name(row_desc); |
38 | 0 | } |
39 | 0 | static MutableBlock build_mutable_mem_reuse_block(Block* block, const RowDescriptor& row_desc) { |
40 | 0 | if (!block->mem_reuse()) { |
41 | 0 | MutableBlock tmp(VectorizedUtils::create_columns_with_type_and_name(row_desc)); |
42 | 0 | block->swap(tmp.to_block()); |
43 | 0 | } |
44 | 0 | return MutableBlock::build_mutable_block(block); |
45 | 0 | } |
46 | 0 | static MutableBlock build_mutable_mem_reuse_block(Block* block, const Block& other) { |
47 | 0 | if (!block->mem_reuse()) { |
48 | 0 | MutableBlock tmp(other.clone_empty()); |
49 | 0 | block->swap(tmp.to_block()); |
50 | 0 | } |
51 | 0 | return MutableBlock::build_mutable_block(block); |
52 | 0 | } |
53 | | static MutableBlock build_mutable_mem_reuse_block(Block* block, |
54 | 0 | std::vector<SlotDescriptor*>& slots) { |
55 | 0 | if (!block->mem_reuse()) { |
56 | 0 | size_t column_size = slots.size(); |
57 | 0 | MutableColumns columns(column_size); |
58 | 0 | for (size_t i = 0; i < column_size; i++) { |
59 | 0 | columns[i] = slots[i]->get_empty_mutable_column(); |
60 | 0 | } |
61 | 0 | int n_columns = 0; |
62 | 0 | for (const auto slot_desc : slots) { |
63 | 0 | block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), |
64 | 0 | slot_desc->get_data_type_ptr(), |
65 | 0 | slot_desc->col_name())); |
66 | 0 | } |
67 | 0 | } |
68 | 0 | return MutableBlock(block); |
69 | 0 | } |
70 | | |
71 | 0 | static ColumnsWithTypeAndName create_columns_with_type_and_name(const RowDescriptor& row_desc) { |
72 | 0 | ColumnsWithTypeAndName columns_with_type_and_name; |
73 | 0 | for (const auto& tuple_desc : row_desc.tuple_descriptors()) { |
74 | 0 | for (const auto& slot_desc : tuple_desc->slots()) { |
75 | 0 | if (!slot_desc->need_materialize()) { |
76 | 0 | continue; |
77 | 0 | } |
78 | 0 | columns_with_type_and_name.emplace_back(nullptr, slot_desc->get_data_type_ptr(), |
79 | 0 | slot_desc->col_name()); |
80 | 0 | } |
81 | 0 | } |
82 | 0 | return columns_with_type_and_name; |
83 | 0 | } |
84 | | |
85 | 0 | static NameAndTypePairs create_name_and_data_types(const RowDescriptor& row_desc) { |
86 | 0 | NameAndTypePairs name_with_types; |
87 | 0 | for (const auto& tuple_desc : row_desc.tuple_descriptors()) { |
88 | 0 | for (const auto& slot_desc : tuple_desc->slots()) { |
89 | 0 | if (!slot_desc->need_materialize()) { |
90 | 0 | continue; |
91 | 0 | } |
92 | 0 | name_with_types.emplace_back(slot_desc->col_name(), slot_desc->get_data_type_ptr()); |
93 | 0 | } |
94 | 0 | } |
95 | 0 | return name_with_types; |
96 | 0 | } |
97 | | |
98 | | static ColumnsWithTypeAndName create_empty_block(const RowDescriptor& row_desc, |
99 | 0 | bool ignore_trivial_slot = true) { |
100 | 0 | ColumnsWithTypeAndName columns_with_type_and_name; |
101 | 0 | for (const auto& tuple_desc : row_desc.tuple_descriptors()) { |
102 | 0 | for (const auto& slot_desc : tuple_desc->slots()) { |
103 | 0 | if (ignore_trivial_slot && !slot_desc->need_materialize()) { |
104 | 0 | continue; |
105 | 0 | } |
106 | 0 | columns_with_type_and_name.emplace_back( |
107 | 0 | slot_desc->get_data_type_ptr()->create_column(), |
108 | 0 | slot_desc->get_data_type_ptr(), slot_desc->col_name()); |
109 | 0 | } |
110 | 0 | } |
111 | 0 | return columns_with_type_and_name; |
112 | 0 | } |
113 | | |
114 | | // is_single: whether src is null map of a ColumnConst |
115 | 730 | static void update_null_map(NullMap& dst, const NullMap& src, bool is_single = false) { |
116 | 730 | size_t size = dst.size(); |
117 | 730 | auto* __restrict l = dst.data(); |
118 | 730 | auto* __restrict r = src.data(); |
119 | 730 | if (is_single) { |
120 | 0 | if (r[0]) { |
121 | 0 | for (size_t i = 0; i < size; ++i) { |
122 | 0 | l[i] = 1; |
123 | 0 | } |
124 | 0 | } |
125 | 730 | } else { |
126 | 5.30k | for (size_t i = 0; i < size; ++i) { |
127 | 4.57k | l[i] |= r[i]; |
128 | 4.57k | } |
129 | 730 | } |
130 | 730 | } |
131 | | |
132 | 0 | static DataTypes get_data_types(const RowDescriptor& row_desc) { |
133 | 0 | DataTypes data_types; |
134 | 0 | for (const auto& tuple_desc : row_desc.tuple_descriptors()) { |
135 | 0 | for (const auto& slot_desc : tuple_desc->slots()) { |
136 | 0 | data_types.push_back(slot_desc->get_data_type_ptr()); |
137 | 0 | } |
138 | 0 | } |
139 | 0 | return data_types; |
140 | 0 | } |
141 | | |
142 | 0 | static std::vector<std::string> get_column_names(const RowDescriptor& row_desc) { |
143 | 0 | std::vector<std::string> column_names; |
144 | 0 | for (const auto& tuple_desc : row_desc.tuple_descriptors()) { |
145 | 0 | for (const auto& slot_desc : tuple_desc->slots()) { |
146 | 0 | column_names.push_back(slot_desc->col_name()); |
147 | 0 | } |
148 | 0 | } |
149 | 0 | return column_names; |
150 | 0 | } |
151 | | |
152 | 20.7k | static bool all_arguments_are_constant(const Block& block, const ColumnNumbers& args) { |
153 | 29.0k | for (const auto& arg : args) { |
154 | 29.0k | if (!is_column_const(*block.get_by_position(arg).column)) { |
155 | 18.7k | return false; |
156 | 18.7k | } |
157 | 29.0k | } |
158 | 2.03k | return true; |
159 | 20.7k | } |
160 | | }; |
161 | | |
162 | 0 | inline bool match_suffix(const std::string& name, const std::string& suffix) { |
163 | 0 | if (name.length() < suffix.length()) { |
164 | 0 | return false; |
165 | 0 | } |
166 | 0 | return name.substr(name.length() - suffix.length()) == suffix; |
167 | 0 | } |
168 | | |
169 | 0 | inline std::string remove_suffix(const std::string& name, const std::string& suffix) { |
170 | 0 | CHECK(match_suffix(name, suffix)) |
171 | 0 | << ", suffix not match, name=" << name << ", suffix=" << suffix; |
172 | 0 | return name.substr(0, name.length() - suffix.length()); |
173 | 0 | }; |
174 | | |
175 | 0 | inline ColumnPtr create_always_true_column(size_t size, bool is_nullable) { |
176 | 0 | ColumnPtr res_data_column = ColumnUInt8::create(1, 1); |
177 | 0 | if (is_nullable) { |
178 | 0 | auto null_map = ColumnVector<UInt8>::create(1, 0); |
179 | 0 | res_data_column = ColumnNullable::create(res_data_column, std::move(null_map)); |
180 | 0 | } |
181 | 0 | return ColumnConst::create(std::move(res_data_column), size); |
182 | 0 | } |
183 | | |
184 | | // change null element to true element |
185 | 0 | inline void change_null_to_true(ColumnPtr column, ColumnPtr argument = nullptr) { |
186 | 0 | size_t rows = column->size(); |
187 | 0 | if (is_column_const(*column)) { |
188 | 0 | change_null_to_true(assert_cast<const ColumnConst*>(column.get())->get_data_column_ptr()); |
189 | 0 | } else if (column->has_null()) { |
190 | 0 | auto* nullable = |
191 | 0 | const_cast<ColumnNullable*>(assert_cast<const ColumnNullable*>(column.get())); |
192 | 0 | auto* __restrict data = assert_cast<ColumnUInt8*>(nullable->get_nested_column_ptr().get()) |
193 | 0 | ->get_data() |
194 | 0 | .data(); |
195 | 0 | auto* __restrict null_map = const_cast<uint8_t*>(nullable->get_null_map_data().data()); |
196 | 0 | for (size_t i = 0; i < rows; ++i) { |
197 | 0 | data[i] |= null_map[i]; |
198 | 0 | } |
199 | 0 | memset(null_map, 0, rows); |
200 | 0 | } else if (argument != nullptr && argument->has_null()) { |
201 | 0 | const auto* __restrict null_map = |
202 | 0 | assert_cast<const ColumnNullable*>(argument.get())->get_null_map_data().data(); |
203 | 0 | auto* __restrict data = |
204 | 0 | const_cast<ColumnUInt8*>(assert_cast<const ColumnUInt8*>(column.get())) |
205 | 0 | ->get_data() |
206 | 0 | .data(); |
207 | 0 | for (size_t i = 0; i < rows; ++i) { |
208 | 0 | data[i] |= null_map[i]; |
209 | 0 | } |
210 | 0 | } |
211 | 0 | } |
212 | | |
213 | 0 | inline size_t calculate_false_number(ColumnPtr column) { |
214 | 0 | size_t rows = column->size(); |
215 | 0 | if (is_column_const(*column)) { |
216 | 0 | return calculate_false_number( |
217 | 0 | assert_cast<const ColumnConst*>(column.get())->get_data_column_ptr()) * |
218 | 0 | rows; |
219 | 0 | } else if (column->is_nullable()) { |
220 | 0 | const auto* nullable = assert_cast<const ColumnNullable*>(column.get()); |
221 | 0 | const auto* data = assert_cast<const ColumnUInt8*>(nullable->get_nested_column_ptr().get()) |
222 | 0 | ->get_data() |
223 | 0 | .data(); |
224 | 0 | const auto* __restrict null_map = nullable->get_null_map_data().data(); |
225 | 0 | return simd::count_zero_num(reinterpret_cast<const int8_t* __restrict>(data), null_map, |
226 | 0 | rows); |
227 | 0 | } else { |
228 | 0 | const auto* data = assert_cast<const ColumnUInt8*>(column.get())->get_data().data(); |
229 | 0 | return simd::count_zero_num(reinterpret_cast<const int8_t* __restrict>(data), rows); |
230 | 0 | } |
231 | 0 | } |
232 | | |
233 | | } // namespace doris::vectorized |
234 | | |
235 | | namespace apache::thrift { |
236 | | template <typename ThriftStruct> |
237 | 2 | ThriftStruct from_json_string(const std::string& json_val) { |
238 | 2 | using namespace apache::thrift::transport; |
239 | 2 | using namespace apache::thrift::protocol; |
240 | 2 | ThriftStruct ts; |
241 | 2 | std::shared_ptr<TTransport> trans = |
242 | 2 | std::make_shared<TMemoryBuffer>((uint8_t*)json_val.c_str(), (uint32_t)json_val.size()); |
243 | 2 | TJSONProtocol protocol(trans); |
244 | 2 | ts.read(&protocol); |
245 | 2 | return ts; |
246 | 2 | } |
247 | | |
248 | | } // namespace apache::thrift |