be/src/exprs/function/function_bitmap.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionBitmap.h |
19 | | // and modified by Doris |
20 | | |
21 | | #include <absl/strings/numbers.h> |
22 | | #include <absl/strings/str_split.h> |
23 | | #include <glog/logging.h> |
24 | | #include <stdint.h> |
25 | | #include <string.h> |
26 | | |
27 | | #include <algorithm> |
28 | | #include <boost/iterator/iterator_facade.hpp> |
29 | | #include <functional> |
30 | | #include <memory> |
31 | | #include <ostream> |
32 | | #include <string> |
33 | | #include <utility> |
34 | | #include <vector> |
35 | | |
36 | | #include "common/compiler_util.h" // IWYU pragma: keep |
37 | | #include "common/status.h" |
38 | | #include "core/assert_cast.h" |
39 | | #include "core/block/block.h" |
40 | | #include "core/block/column_numbers.h" |
41 | | #include "core/block/column_with_type_and_name.h" |
42 | | #include "core/column/column.h" |
43 | | #include "core/column/column_array.h" |
44 | | #include "core/column/column_complex.h" |
45 | | #include "core/column/column_const.h" |
46 | | #include "core/column/column_nullable.h" |
47 | | #include "core/column/column_string.h" |
48 | | #include "core/column/column_vector.h" |
49 | | #include "core/data_type/data_type.h" |
50 | | #include "core/data_type/data_type_array.h" |
51 | | #include "core/data_type/data_type_bitmap.h" |
52 | | #include "core/data_type/data_type_nullable.h" |
53 | | #include "core/data_type/data_type_number.h" |
54 | | #include "core/data_type/data_type_string.h" |
55 | | #include "core/field.h" |
56 | | #include "core/types.h" |
57 | | #include "core/value/bitmap_value.h" |
58 | | #include "exec/common/stringop_substring.h" |
59 | | #include "exec/common/util.hpp" |
60 | | #include "exprs/aggregate/aggregate_function.h" |
61 | | #include "exprs/function/function.h" |
62 | | #include "exprs/function/function_always_not_nullable.h" |
63 | | #include "exprs/function/function_bitmap_min_or_max.h" |
64 | | #include "exprs/function/function_const.h" |
65 | | #include "exprs/function/function_helpers.h" |
66 | | #include "exprs/function/function_totype.h" |
67 | | #include "exprs/function/simple_function_factory.h" |
68 | | #include "util/hash/murmur_hash3.h" |
69 | | #include "util/hash_util.hpp" |
70 | | #include "util/string_parser.hpp" |
71 | | #include "util/url_coding.h" |
72 | | |
73 | | namespace doris { |
74 | | class FunctionContext; |
75 | | } // namespace doris |
76 | | |
77 | | namespace doris { |
78 | | |
79 | | struct BitmapEmpty { |
80 | | static constexpr auto name = "bitmap_empty"; |
81 | | using ReturnColVec = ColumnBitmap; |
82 | 272 | static DataTypePtr get_return_type() { return std::make_shared<DataTypeBitMap>(); } |
83 | 272 | static auto init_value() { return BitmapValue {}; } |
84 | | }; |
85 | | |
86 | | struct ToBitmap { |
87 | | static constexpr auto name = "to_bitmap"; |
88 | | using ReturnType = DataTypeBitMap; |
89 | | |
90 | | template <typename ColumnType> |
91 | 1.08k | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { |
92 | 1.08k | execute<ColumnType, false>(col, nullptr, col_res); |
93 | 1.08k | } _ZN5doris8ToBitmap6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS8_EE Line | Count | Source | 91 | 12 | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { | 92 | 12 | execute<ColumnType, false>(col, nullptr, col_res); | 93 | 12 | } |
_ZN5doris8ToBitmap6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE Line | Count | Source | 91 | 1.07k | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { | 92 | 1.07k | execute<ColumnType, false>(col, nullptr, col_res); | 93 | 1.07k | } |
|
94 | | template <typename ColumnType> |
95 | | static void vector_nullable(const ColumnType* col, const NullMap& nullmap, |
96 | 532 | MutableColumnPtr& col_res) { |
97 | 532 | execute<ColumnType, true>(col, &nullmap, col_res); |
98 | 532 | } _ZN5doris8ToBitmap15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE Line | Count | Source | 96 | 51 | MutableColumnPtr& col_res) { | 97 | 51 | execute<ColumnType, true>(col, &nullmap, col_res); | 98 | 51 | } |
_ZN5doris8ToBitmap15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE Line | Count | Source | 96 | 481 | MutableColumnPtr& col_res) { | 97 | 481 | execute<ColumnType, true>(col, &nullmap, col_res); | 98 | 481 | } |
|
99 | | template <typename ColumnType, bool arg_is_nullable> |
100 | 1.61k | static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) { |
101 | 1.61k | if constexpr (std::is_same_v<ColumnType, ColumnString>) { |
102 | 63 | const ColumnString::Chars& data = col->get_chars(); |
103 | 63 | const ColumnString::Offsets& offsets = col->get_offsets(); |
104 | | |
105 | 63 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
106 | 63 | auto& res_data = res_column->get_data(); |
107 | 63 | size_t size = offsets.size(); |
108 | | |
109 | 3.57k | for (size_t i = 0; i < size; ++i) { |
110 | 3.50k | if (arg_is_nullable && ((*nullmap)[i])) { |
111 | 34 | continue; |
112 | 3.47k | } else { |
113 | 3.47k | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
114 | 3.47k | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); |
115 | 3.47k | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; |
116 | 3.47k | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( |
117 | 3.47k | raw_str, str_size, &parse_result); |
118 | 3.47k | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { |
119 | 3.26k | res_data[i].add(int_value); |
120 | 3.26k | } |
121 | 3.47k | } |
122 | 3.50k | } |
123 | 1.55k | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { |
124 | 1.55k | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
125 | 1.55k | auto& res_data = res_column->get_data(); |
126 | 1.55k | size_t size = col->size(); |
127 | | |
128 | 141k | for (size_t i = 0; i < size; ++i) { |
129 | 139k | if constexpr (arg_is_nullable) { |
130 | 1.94k | if ((*nullmap)[i]) { |
131 | 830 | continue; |
132 | 830 | } |
133 | 1.94k | } |
134 | 139k | if (auto value = col->get_data()[i]; value >= 0) { |
135 | 133k | res_data[i].add(value); |
136 | 133k | } |
137 | 139k | } |
138 | 1.55k | } |
139 | 1.61k | } _ZN5doris8ToBitmap7executeINS_9ColumnStrIjEELb1EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE Line | Count | Source | 100 | 51 | static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) { | 101 | 51 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 102 | 51 | const ColumnString::Chars& data = col->get_chars(); | 103 | 51 | const ColumnString::Offsets& offsets = col->get_offsets(); | 104 | | | 105 | 51 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 106 | 51 | auto& res_data = res_column->get_data(); | 107 | 51 | size_t size = offsets.size(); | 108 | | | 109 | 3.53k | for (size_t i = 0; i < size; ++i) { | 110 | 3.47k | if (arg_is_nullable && ((*nullmap)[i])) { | 111 | 34 | continue; | 112 | 3.44k | } else { | 113 | 3.44k | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 114 | 3.44k | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 115 | 3.44k | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 116 | 3.44k | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 117 | 3.44k | raw_str, str_size, &parse_result); | 118 | 3.44k | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 119 | 3.26k | res_data[i].add(int_value); | 120 | 3.26k | } | 121 | 3.44k | } | 122 | 3.47k | } | 123 | | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 124 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 125 | | auto& res_data = res_column->get_data(); | 126 | | size_t size = col->size(); | 127 | | | 128 | | for (size_t i = 0; i < size; ++i) { | 129 | | if constexpr (arg_is_nullable) { | 130 | | if ((*nullmap)[i]) { | 131 | | continue; | 132 | | } | 133 | | } | 134 | | if (auto value = col->get_data()[i]; value >= 0) { | 135 | | res_data[i].add(value); | 136 | | } | 137 | | } | 138 | | } | 139 | 51 | } |
_ZN5doris8ToBitmap7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb1EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE Line | Count | Source | 100 | 481 | static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) { | 101 | | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 102 | | const ColumnString::Chars& data = col->get_chars(); | 103 | | const ColumnString::Offsets& offsets = col->get_offsets(); | 104 | | | 105 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 106 | | auto& res_data = res_column->get_data(); | 107 | | size_t size = offsets.size(); | 108 | | | 109 | | for (size_t i = 0; i < size; ++i) { | 110 | | if (arg_is_nullable && ((*nullmap)[i])) { | 111 | | continue; | 112 | | } else { | 113 | | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 114 | | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 115 | | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 116 | | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 117 | | raw_str, str_size, &parse_result); | 118 | | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 119 | | res_data[i].add(int_value); | 120 | | } | 121 | | } | 122 | | } | 123 | 481 | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 124 | 481 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 125 | 481 | auto& res_data = res_column->get_data(); | 126 | 481 | size_t size = col->size(); | 127 | | | 128 | 3.26k | for (size_t i = 0; i < size; ++i) { | 129 | 1.94k | if constexpr (arg_is_nullable) { | 130 | 1.94k | if ((*nullmap)[i]) { | 131 | 830 | continue; | 132 | 830 | } | 133 | 1.94k | } | 134 | 1.94k | if (auto value = col->get_data()[i]; value >= 0) { | 135 | 785 | res_data[i].add(value); | 136 | 785 | } | 137 | 1.94k | } | 138 | 481 | } | 139 | 481 | } |
_ZN5doris8ToBitmap7executeINS_9ColumnStrIjEELb0EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE Line | Count | Source | 100 | 12 | static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) { | 101 | 12 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 102 | 12 | const ColumnString::Chars& data = col->get_chars(); | 103 | 12 | const ColumnString::Offsets& offsets = col->get_offsets(); | 104 | | | 105 | 12 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 106 | 12 | auto& res_data = res_column->get_data(); | 107 | 12 | size_t size = offsets.size(); | 108 | | | 109 | 40 | for (size_t i = 0; i < size; ++i) { | 110 | 28 | if (arg_is_nullable && ((*nullmap)[i])) { | 111 | 0 | continue; | 112 | 28 | } else { | 113 | 28 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 114 | 28 | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 115 | 28 | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 116 | 28 | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 117 | 28 | raw_str, str_size, &parse_result); | 118 | 28 | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 119 | 0 | res_data[i].add(int_value); | 120 | 0 | } | 121 | 28 | } | 122 | 28 | } | 123 | | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 124 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 125 | | auto& res_data = res_column->get_data(); | 126 | | size_t size = col->size(); | 127 | | | 128 | | for (size_t i = 0; i < size; ++i) { | 129 | | if constexpr (arg_is_nullable) { | 130 | | if ((*nullmap)[i]) { | 131 | | continue; | 132 | | } | 133 | | } | 134 | | if (auto value = col->get_data()[i]; value >= 0) { | 135 | | res_data[i].add(value); | 136 | | } | 137 | | } | 138 | | } | 139 | 12 | } |
_ZN5doris8ToBitmap7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb0EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE Line | Count | Source | 100 | 1.07k | static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) { | 101 | | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 102 | | const ColumnString::Chars& data = col->get_chars(); | 103 | | const ColumnString::Offsets& offsets = col->get_offsets(); | 104 | | | 105 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 106 | | auto& res_data = res_column->get_data(); | 107 | | size_t size = offsets.size(); | 108 | | | 109 | | for (size_t i = 0; i < size; ++i) { | 110 | | if (arg_is_nullable && ((*nullmap)[i])) { | 111 | | continue; | 112 | | } else { | 113 | | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 114 | | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 115 | | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 116 | | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 117 | | raw_str, str_size, &parse_result); | 118 | | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 119 | | res_data[i].add(int_value); | 120 | | } | 121 | | } | 122 | | } | 123 | 1.07k | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 124 | 1.07k | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 125 | 1.07k | auto& res_data = res_column->get_data(); | 126 | 1.07k | size_t size = col->size(); | 127 | | | 128 | 138k | for (size_t i = 0; i < size; ++i) { | 129 | | if constexpr (arg_is_nullable) { | 130 | | if ((*nullmap)[i]) { | 131 | | continue; | 132 | | } | 133 | | } | 134 | 137k | if (auto value = col->get_data()[i]; value >= 0) { | 135 | 133k | res_data[i].add(value); | 136 | 133k | } | 137 | 137k | } | 138 | 1.07k | } | 139 | 1.07k | } |
|
140 | | }; |
141 | | |
142 | | struct ToBitmapWithCheck { |
143 | | static constexpr auto name = "to_bitmap_with_check"; |
144 | | using ReturnType = DataTypeBitMap; |
145 | | |
146 | | template <typename ColumnType> |
147 | 52 | static Status vector(const ColumnType* col, MutableColumnPtr& col_res) { |
148 | 52 | return execute<ColumnType, false>(col, nullptr, col_res); |
149 | 52 | } Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck6vectorINS_9ColumnStrIjEEEENS_6StatusEPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE _ZN5doris17ToBitmapWithCheck6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusEPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE Line | Count | Source | 147 | 52 | static Status vector(const ColumnType* col, MutableColumnPtr& col_res) { | 148 | 52 | return execute<ColumnType, false>(col, nullptr, col_res); | 149 | 52 | } |
|
150 | | template <typename ColumnType> |
151 | | static Status vector_nullable(const ColumnType* col, const NullMap& nullmap, |
152 | 51 | MutableColumnPtr& col_res) { |
153 | 51 | return execute<ColumnType, true>(col, &nullmap, col_res); |
154 | 51 | } Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck15vector_nullableINS_9ColumnStrIjEEEENS_6StatusEPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE _ZN5doris17ToBitmapWithCheck15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusEPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE Line | Count | Source | 152 | 51 | MutableColumnPtr& col_res) { | 153 | 51 | return execute<ColumnType, true>(col, &nullmap, col_res); | 154 | 51 | } |
|
155 | | template <typename ColumnType, bool arg_is_nullable> |
156 | | static Status execute(const ColumnType* col, const NullMap* nullmap, |
157 | 103 | MutableColumnPtr& col_res) { |
158 | 103 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { |
159 | 0 | const ColumnString::Chars& data = col->get_chars(); |
160 | 0 | const ColumnString::Offsets& offsets = col->get_offsets(); |
161 | 0 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
162 | 0 | auto& res_data = res_column->get_data(); |
163 | 0 | size_t size = offsets.size(); |
164 | |
|
165 | 0 | for (size_t i = 0; i < size; ++i) { |
166 | 0 | if (arg_is_nullable && ((*nullmap)[i])) { |
167 | 0 | continue; |
168 | 0 | } else { |
169 | 0 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
170 | | // The string lenght is less than 2G, so that cast the str size to int, not use size_t |
171 | 0 | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); |
172 | 0 | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; |
173 | 0 | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( |
174 | 0 | raw_str, str_size, &parse_result); |
175 | 0 | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { |
176 | 0 | res_data[i].add(int_value); |
177 | 0 | } else { |
178 | 0 | return Status::InvalidArgument( |
179 | 0 | "The input: {} is not valid, to_bitmap only support bigint value " |
180 | 0 | "from 0 to 18446744073709551615 currently, cannot create MV with " |
181 | 0 | "to_bitmap on column with negative values or cannot load negative " |
182 | 0 | "values to column with to_bitmap MV on it.", |
183 | 0 | std::string(raw_str, str_size)); |
184 | 0 | } |
185 | 0 | } |
186 | 0 | } |
187 | 103 | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { |
188 | 103 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
189 | 103 | auto& res_data = res_column->get_data(); |
190 | 103 | size_t size = col->size(); |
191 | | |
192 | 230 | for (size_t i = 0; i < size; ++i) { |
193 | 130 | if (arg_is_nullable && ((*nullmap)[i])) { |
194 | 21 | continue; |
195 | 109 | } else { |
196 | 109 | int64_t int_value = col->get_data()[i]; |
197 | 109 | if (LIKELY(int_value >= 0)) { |
198 | 106 | res_data[i].add(int_value); |
199 | 106 | } else { |
200 | 3 | return Status::InvalidArgument( |
201 | 3 | "The input: {} is not valid, to_bitmap only support bigint value " |
202 | 3 | "from 0 to 18446744073709551615 currently, cannot create MV with " |
203 | 3 | "to_bitmap on column with negative values or cannot load negative " |
204 | 3 | "values to column with to_bitmap MV on it.", |
205 | 3 | int_value); |
206 | 3 | } |
207 | 109 | } |
208 | 130 | } |
209 | | } else { |
210 | | return Status::InvalidArgument("not support type"); |
211 | | } |
212 | 100 | return Status::OK(); |
213 | 103 | } Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck7executeINS_9ColumnStrIjEELb1EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE _ZN5doris17ToBitmapWithCheck7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb1EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE Line | Count | Source | 157 | 51 | MutableColumnPtr& col_res) { | 158 | | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 159 | | const ColumnString::Chars& data = col->get_chars(); | 160 | | const ColumnString::Offsets& offsets = col->get_offsets(); | 161 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 162 | | auto& res_data = res_column->get_data(); | 163 | | size_t size = offsets.size(); | 164 | | | 165 | | for (size_t i = 0; i < size; ++i) { | 166 | | if (arg_is_nullable && ((*nullmap)[i])) { | 167 | | continue; | 168 | | } else { | 169 | | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 170 | | // The string lenght is less than 2G, so that cast the str size to int, not use size_t | 171 | | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 172 | | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 173 | | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 174 | | raw_str, str_size, &parse_result); | 175 | | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 176 | | res_data[i].add(int_value); | 177 | | } else { | 178 | | return Status::InvalidArgument( | 179 | | "The input: {} is not valid, to_bitmap only support bigint value " | 180 | | "from 0 to 18446744073709551615 currently, cannot create MV with " | 181 | | "to_bitmap on column with negative values or cannot load negative " | 182 | | "values to column with to_bitmap MV on it.", | 183 | | std::string(raw_str, str_size)); | 184 | | } | 185 | | } | 186 | | } | 187 | 51 | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 188 | 51 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 189 | 51 | auto& res_data = res_column->get_data(); | 190 | 51 | size_t size = col->size(); | 191 | | | 192 | 121 | for (size_t i = 0; i < size; ++i) { | 193 | 70 | if (arg_is_nullable && ((*nullmap)[i])) { | 194 | 21 | continue; | 195 | 49 | } else { | 196 | 49 | int64_t int_value = col->get_data()[i]; | 197 | 49 | if (LIKELY(int_value >= 0)) { | 198 | 49 | res_data[i].add(int_value); | 199 | 49 | } else { | 200 | 0 | return Status::InvalidArgument( | 201 | 0 | "The input: {} is not valid, to_bitmap only support bigint value " | 202 | 0 | "from 0 to 18446744073709551615 currently, cannot create MV with " | 203 | 0 | "to_bitmap on column with negative values or cannot load negative " | 204 | 0 | "values to column with to_bitmap MV on it.", | 205 | 0 | int_value); | 206 | 0 | } | 207 | 49 | } | 208 | 70 | } | 209 | | } else { | 210 | | return Status::InvalidArgument("not support type"); | 211 | | } | 212 | 51 | return Status::OK(); | 213 | 51 | } |
Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck7executeINS_9ColumnStrIjEELb0EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE _ZN5doris17ToBitmapWithCheck7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb0EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE Line | Count | Source | 157 | 52 | MutableColumnPtr& col_res) { | 158 | | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 159 | | const ColumnString::Chars& data = col->get_chars(); | 160 | | const ColumnString::Offsets& offsets = col->get_offsets(); | 161 | | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 162 | | auto& res_data = res_column->get_data(); | 163 | | size_t size = offsets.size(); | 164 | | | 165 | | for (size_t i = 0; i < size; ++i) { | 166 | | if (arg_is_nullable && ((*nullmap)[i])) { | 167 | | continue; | 168 | | } else { | 169 | | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 170 | | // The string lenght is less than 2G, so that cast the str size to int, not use size_t | 171 | | int str_size = cast_set<int>(offsets[i] - offsets[i - 1]); | 172 | | StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; | 173 | | uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>( | 174 | | raw_str, str_size, &parse_result); | 175 | | if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { | 176 | | res_data[i].add(int_value); | 177 | | } else { | 178 | | return Status::InvalidArgument( | 179 | | "The input: {} is not valid, to_bitmap only support bigint value " | 180 | | "from 0 to 18446744073709551615 currently, cannot create MV with " | 181 | | "to_bitmap on column with negative values or cannot load negative " | 182 | | "values to column with to_bitmap MV on it.", | 183 | | std::string(raw_str, str_size)); | 184 | | } | 185 | | } | 186 | | } | 187 | 52 | } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) { | 188 | 52 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 189 | 52 | auto& res_data = res_column->get_data(); | 190 | 52 | size_t size = col->size(); | 191 | | | 192 | 109 | for (size_t i = 0; i < size; ++i) { | 193 | 60 | if (arg_is_nullable && ((*nullmap)[i])) { | 194 | 0 | continue; | 195 | 60 | } else { | 196 | 60 | int64_t int_value = col->get_data()[i]; | 197 | 60 | if (LIKELY(int_value >= 0)) { | 198 | 57 | res_data[i].add(int_value); | 199 | 57 | } else { | 200 | 3 | return Status::InvalidArgument( | 201 | 3 | "The input: {} is not valid, to_bitmap only support bigint value " | 202 | 3 | "from 0 to 18446744073709551615 currently, cannot create MV with " | 203 | 3 | "to_bitmap on column with negative values or cannot load negative " | 204 | 3 | "values to column with to_bitmap MV on it.", | 205 | 3 | int_value); | 206 | 3 | } | 207 | 60 | } | 208 | 60 | } | 209 | | } else { | 210 | | return Status::InvalidArgument("not support type"); | 211 | | } | 212 | 49 | return Status::OK(); | 213 | 52 | } |
|
214 | | }; |
215 | | |
216 | | struct BitmapFromString { |
217 | | using ArgumentType = DataTypeString; |
218 | | |
219 | | static constexpr auto name = "bitmap_from_string"; |
220 | | |
221 | | static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, |
222 | | std::vector<BitmapValue>& res, NullMap& null_map, |
223 | 281 | size_t input_rows_count) { |
224 | 281 | res.reserve(input_rows_count); |
225 | 281 | std::vector<uint64_t> bits; |
226 | 281 | if (offsets.size() == 0 && input_rows_count == 1) { |
227 | | // For NULL constant |
228 | 0 | res.emplace_back(); |
229 | 0 | null_map[0] = 1; |
230 | 0 | return Status::OK(); |
231 | 0 | } |
232 | | |
233 | 370 | auto split_and_parse = [&bits](const char* raw_str, size_t str_size) { |
234 | 370 | bits.clear(); |
235 | 370 | auto res = absl::StrSplit(std::string_view {raw_str, str_size}, ",", absl::SkipEmpty()); |
236 | 370 | uint64_t value = 0; |
237 | 1.28k | for (auto s : res) { |
238 | 1.28k | if (!absl::SimpleAtoi(s, &value)) { |
239 | 66 | return false; |
240 | 66 | } |
241 | 1.22k | bits.push_back(value); |
242 | 1.22k | } |
243 | 304 | return true; |
244 | 370 | }; |
245 | | |
246 | | // split by comma |
247 | | |
248 | 651 | for (size_t i = 0; i < input_rows_count; ++i) { |
249 | 370 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
250 | 370 | int64_t str_size = offsets[i] - offsets[i - 1]; |
251 | | |
252 | 370 | if ((str_size > INT32_MAX) || !split_and_parse(raw_str, str_size)) { |
253 | 66 | res.emplace_back(); |
254 | 66 | null_map[i] = 1; |
255 | 66 | continue; |
256 | 66 | } |
257 | 304 | res.emplace_back(bits); |
258 | 304 | } |
259 | 281 | return Status::OK(); |
260 | 281 | } |
261 | | }; |
262 | | |
263 | | struct NameBitmapFromBase64 { |
264 | | static constexpr auto name = "bitmap_from_base64"; |
265 | | }; |
266 | | struct BitmapFromBase64 { |
267 | | using ArgumentType = DataTypeString; |
268 | | |
269 | | static constexpr auto name = "bitmap_from_base64"; |
270 | | |
271 | | static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, |
272 | | std::vector<BitmapValue>& res, NullMap& null_map, |
273 | 25 | size_t input_rows_count) { |
274 | 25 | res.reserve(input_rows_count); |
275 | 25 | if (offsets.size() == 0 && input_rows_count == 1) { |
276 | | // For NULL constant |
277 | 0 | res.emplace_back(); |
278 | 0 | null_map[0] = 1; |
279 | 0 | return Status::OK(); |
280 | 0 | } |
281 | 25 | std::string decode_buff; |
282 | 25 | size_t last_decode_buff_len = 0; |
283 | 25 | size_t curr_decode_buff_len = 0; |
284 | 73 | for (size_t i = 0; i < input_rows_count; ++i) { |
285 | 49 | const char* src_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
286 | 49 | size_t src_size = offsets[i] - offsets[i - 1]; |
287 | 49 | if (0 != src_size % 4) { |
288 | | // return Status::InvalidArgument( |
289 | | // fmt::format("invalid base64: {}", std::string(src_str, src_size))); |
290 | 0 | res.emplace_back(); |
291 | 0 | null_map[i] = 1; |
292 | 0 | continue; |
293 | 0 | } |
294 | 49 | curr_decode_buff_len = src_size + 3; |
295 | 49 | if (curr_decode_buff_len > last_decode_buff_len) { |
296 | 37 | decode_buff.resize(curr_decode_buff_len); |
297 | 37 | last_decode_buff_len = curr_decode_buff_len; |
298 | 37 | } |
299 | 49 | auto outlen = base64_decode(src_str, src_size, decode_buff.data()); |
300 | 49 | if (outlen < 0) { |
301 | 0 | res.emplace_back(); |
302 | 0 | null_map[i] = 1; |
303 | 49 | } else { |
304 | 49 | BitmapValue bitmap_val; |
305 | 49 | if (!bitmap_val.deserialize(decode_buff.data())) { |
306 | 1 | return Status::RuntimeError("bitmap_from_base64 decode failed: base64: {}", |
307 | 1 | std::string(src_str, src_size)); |
308 | 1 | } |
309 | 48 | res.emplace_back(std::move(bitmap_val)); |
310 | 48 | } |
311 | 49 | } |
312 | 24 | return Status::OK(); |
313 | 25 | } |
314 | | }; |
315 | | struct BitmapFromArray { |
316 | | using ArgumentType = DataTypeArray; |
317 | | static constexpr auto name = "bitmap_from_array"; |
318 | | |
319 | | template <typename ColumnType> |
320 | | static Status vector(const ColumnArray::Offsets64& offset_column_data, |
321 | | const IColumn& nested_column, const NullMap& nested_null_map, |
322 | 13 | std::vector<BitmapValue>& res, NullMap& null_map) { |
323 | 13 | const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data(); |
324 | 13 | auto size = offset_column_data.size(); |
325 | 13 | res.reserve(size); |
326 | | // Preserve the nested column's native integer type here. |
327 | | // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path |
328 | | // instead of widening every element to uint64_t first. |
329 | 13 | using ValueType = typename ColumnType::value_type; |
330 | 13 | std::vector<ValueType> bits; |
331 | 36 | for (size_t i = 0; i < size; ++i) { |
332 | 23 | auto curr_offset = offset_column_data[i]; |
333 | 23 | auto prev_offset = offset_column_data[i - 1]; |
334 | 23 | bits.reserve(curr_offset - prev_offset); |
335 | 92 | for (auto j = prev_offset; j < curr_offset; ++j) { |
336 | 71 | auto data = nested_column_data[j]; |
337 | | // invaild value |
338 | 71 | if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) { |
339 | 2 | res.emplace_back(); |
340 | 2 | null_map[i] = 1; |
341 | 2 | break; |
342 | 69 | } else { |
343 | 69 | bits.push_back(data); |
344 | 69 | } |
345 | 71 | } |
346 | | //input is valid value |
347 | 23 | if (!null_map[i]) { |
348 | 21 | res.emplace_back(bits); |
349 | 21 | } |
350 | 23 | bits.clear(); |
351 | 23 | } |
352 | 13 | return Status::OK(); |
353 | 13 | } _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE3EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_ Line | Count | Source | 322 | 3 | std::vector<BitmapValue>& res, NullMap& null_map) { | 323 | 3 | const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data(); | 324 | 3 | auto size = offset_column_data.size(); | 325 | 3 | res.reserve(size); | 326 | | // Preserve the nested column's native integer type here. | 327 | | // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path | 328 | | // instead of widening every element to uint64_t first. | 329 | 3 | using ValueType = typename ColumnType::value_type; | 330 | 3 | std::vector<ValueType> bits; | 331 | 6 | for (size_t i = 0; i < size; ++i) { | 332 | 3 | auto curr_offset = offset_column_data[i]; | 333 | 3 | auto prev_offset = offset_column_data[i - 1]; | 334 | 3 | bits.reserve(curr_offset - prev_offset); | 335 | 16 | for (auto j = prev_offset; j < curr_offset; ++j) { | 336 | 13 | auto data = nested_column_data[j]; | 337 | | // invaild value | 338 | 13 | if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) { | 339 | 0 | res.emplace_back(); | 340 | 0 | null_map[i] = 1; | 341 | 0 | break; | 342 | 13 | } else { | 343 | 13 | bits.push_back(data); | 344 | 13 | } | 345 | 13 | } | 346 | | //input is valid value | 347 | 3 | if (!null_map[i]) { | 348 | 3 | res.emplace_back(bits); | 349 | 3 | } | 350 | 3 | bits.clear(); | 351 | 3 | } | 352 | 3 | return Status::OK(); | 353 | 3 | } |
Unexecuted instantiation: _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE2EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_ Unexecuted instantiation: _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE4EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_ _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE5EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_ Line | Count | Source | 322 | 3 | std::vector<BitmapValue>& res, NullMap& null_map) { | 323 | 3 | const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data(); | 324 | 3 | auto size = offset_column_data.size(); | 325 | 3 | res.reserve(size); | 326 | | // Preserve the nested column's native integer type here. | 327 | | // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path | 328 | | // instead of widening every element to uint64_t first. | 329 | 3 | using ValueType = typename ColumnType::value_type; | 330 | 3 | std::vector<ValueType> bits; | 331 | 14 | for (size_t i = 0; i < size; ++i) { | 332 | 11 | auto curr_offset = offset_column_data[i]; | 333 | 11 | auto prev_offset = offset_column_data[i - 1]; | 334 | 11 | bits.reserve(curr_offset - prev_offset); | 335 | 56 | for (auto j = prev_offset; j < curr_offset; ++j) { | 336 | 47 | auto data = nested_column_data[j]; | 337 | | // invaild value | 338 | 47 | if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) { | 339 | 2 | res.emplace_back(); | 340 | 2 | null_map[i] = 1; | 341 | 2 | break; | 342 | 45 | } else { | 343 | 45 | bits.push_back(data); | 344 | 45 | } | 345 | 47 | } | 346 | | //input is valid value | 347 | 11 | if (!null_map[i]) { | 348 | 9 | res.emplace_back(bits); | 349 | 9 | } | 350 | 11 | bits.clear(); | 351 | 11 | } | 352 | 3 | return Status::OK(); | 353 | 3 | } |
_ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_ Line | Count | Source | 322 | 7 | std::vector<BitmapValue>& res, NullMap& null_map) { | 323 | 7 | const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data(); | 324 | 7 | auto size = offset_column_data.size(); | 325 | 7 | res.reserve(size); | 326 | | // Preserve the nested column's native integer type here. | 327 | | // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path | 328 | | // instead of widening every element to uint64_t first. | 329 | 7 | using ValueType = typename ColumnType::value_type; | 330 | 7 | std::vector<ValueType> bits; | 331 | 16 | for (size_t i = 0; i < size; ++i) { | 332 | 9 | auto curr_offset = offset_column_data[i]; | 333 | 9 | auto prev_offset = offset_column_data[i - 1]; | 334 | 9 | bits.reserve(curr_offset - prev_offset); | 335 | 20 | for (auto j = prev_offset; j < curr_offset; ++j) { | 336 | 11 | auto data = nested_column_data[j]; | 337 | | // invaild value | 338 | 11 | if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) { | 339 | 0 | res.emplace_back(); | 340 | 0 | null_map[i] = 1; | 341 | 0 | break; | 342 | 11 | } else { | 343 | 11 | bits.push_back(data); | 344 | 11 | } | 345 | 11 | } | 346 | | //input is valid value | 347 | 9 | if (!null_map[i]) { | 348 | 9 | res.emplace_back(bits); | 349 | 9 | } | 350 | 9 | bits.clear(); | 351 | 9 | } | 352 | 7 | return Status::OK(); | 353 | 7 | } |
|
354 | | }; |
355 | | |
356 | | template <typename Impl> |
357 | | class FunctionBitmapAlwaysNull : public IFunction { |
358 | | public: |
359 | | static constexpr auto name = Impl::name; |
360 | | |
361 | 3 | String get_name() const override { return name; }_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE8get_nameB5cxx11Ev Line | Count | Source | 361 | 1 | String get_name() const override { return name; } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE8get_nameB5cxx11Ev Line | Count | Source | 361 | 1 | String get_name() const override { return name; } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE8get_nameB5cxx11Ev Line | Count | Source | 361 | 1 | String get_name() const override { return name; } |
|
362 | | |
363 | 332 | static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); }_ZN5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE6createEv Line | Count | Source | 363 | 276 | static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); } |
_ZN5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE6createEv Line | Count | Source | 363 | 33 | static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); } |
_ZN5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE6createEv Line | Count | Source | 363 | 23 | static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); } |
|
364 | | |
365 | 305 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { |
366 | 305 | return make_nullable(std::make_shared<DataTypeBitMap>()); |
367 | 305 | } _ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 365 | 267 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 366 | 267 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 367 | 267 | } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 365 | 24 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 366 | 24 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 367 | 24 | } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 365 | 14 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 366 | 14 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 367 | 14 | } |
|
368 | | |
369 | 305 | size_t get_number_of_arguments() const override { return 1; }_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE23get_number_of_argumentsEv Line | Count | Source | 369 | 267 | size_t get_number_of_arguments() const override { return 1; } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE23get_number_of_argumentsEv Line | Count | Source | 369 | 24 | size_t get_number_of_arguments() const override { return 1; } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE23get_number_of_argumentsEv Line | Count | Source | 369 | 14 | size_t get_number_of_arguments() const override { return 1; } |
|
370 | | |
371 | | Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
372 | 319 | uint32_t result, size_t input_rows_count) const override { |
373 | 319 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); |
374 | 319 | auto res_data_column = ColumnBitmap::create(); |
375 | 319 | auto& null_map = res_null_map->get_data(); |
376 | 319 | auto& res = res_data_column->get_data(); |
377 | | |
378 | 319 | ColumnPtr& argument_column = block.get_by_position(arguments[0]).column; |
379 | 319 | if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) { |
380 | 306 | const auto& str_column = static_cast<const ColumnString&>(*argument_column); |
381 | 306 | const ColumnString::Chars& data = str_column.get_chars(); |
382 | 306 | const ColumnString::Offsets& offsets = str_column.get_offsets(); |
383 | 306 | RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count)); |
384 | 306 | } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) { |
385 | 13 | auto argument_type = remove_nullable( |
386 | 13 | assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type) |
387 | 13 | .get_nested_type()); |
388 | 13 | const auto& array_column = static_cast<const ColumnArray&>(*argument_column); |
389 | 13 | const auto& offset_column_data = array_column.get_offsets(); |
390 | 13 | const auto& nested_nullable_column = |
391 | 13 | static_cast<const ColumnNullable&>(array_column.get_data()); |
392 | 13 | const auto& nested_column = nested_nullable_column.get_nested_column(); |
393 | 13 | const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data(); |
394 | | |
395 | 13 | switch (argument_type->get_primitive_type()) { |
396 | 3 | case PrimitiveType::TYPE_TINYINT: |
397 | 3 | RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column, |
398 | 3 | nested_null_map, res, null_map)); |
399 | 3 | break; |
400 | 3 | case PrimitiveType::TYPE_BOOLEAN: |
401 | 0 | RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>( |
402 | 0 | offset_column_data, nested_column, nested_null_map, res, null_map)); |
403 | 0 | break; |
404 | 0 | case PrimitiveType::TYPE_SMALLINT: |
405 | 0 | RETURN_IF_ERROR(Impl::template vector<ColumnInt16>( |
406 | 0 | offset_column_data, nested_column, nested_null_map, res, null_map)); |
407 | 0 | break; |
408 | 3 | case PrimitiveType::TYPE_INT: |
409 | 3 | RETURN_IF_ERROR(Impl::template vector<ColumnInt32>( |
410 | 3 | offset_column_data, nested_column, nested_null_map, res, null_map)); |
411 | 3 | break; |
412 | 7 | case PrimitiveType::TYPE_BIGINT: |
413 | 7 | RETURN_IF_ERROR(Impl::template vector<ColumnInt64>( |
414 | 7 | offset_column_data, nested_column, nested_null_map, res, null_map)); |
415 | 7 | break; |
416 | 7 | default: |
417 | 0 | return Status::RuntimeError("Illegal column {} of argument of function {}", |
418 | 0 | block.get_by_position(arguments[0]).column->get_name(), |
419 | 0 | get_name()); |
420 | 13 | } |
421 | | } else { |
422 | | return Status::RuntimeError("Illegal column {} of argument of function {}", |
423 | | block.get_by_position(arguments[0]).column->get_name(), |
424 | | get_name()); |
425 | | } |
426 | 318 | block.get_by_position(result).column = |
427 | 319 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); |
428 | 319 | return Status::OK(); |
429 | 319 | } _ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 372 | 281 | uint32_t result, size_t input_rows_count) const override { | 373 | 281 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 374 | 281 | auto res_data_column = ColumnBitmap::create(); | 375 | 281 | auto& null_map = res_null_map->get_data(); | 376 | 281 | auto& res = res_data_column->get_data(); | 377 | | | 378 | 281 | ColumnPtr& argument_column = block.get_by_position(arguments[0]).column; | 379 | 281 | if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) { | 380 | 281 | const auto& str_column = static_cast<const ColumnString&>(*argument_column); | 381 | 281 | const ColumnString::Chars& data = str_column.get_chars(); | 382 | 281 | const ColumnString::Offsets& offsets = str_column.get_offsets(); | 383 | 281 | RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count)); | 384 | | } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) { | 385 | | auto argument_type = remove_nullable( | 386 | | assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type) | 387 | | .get_nested_type()); | 388 | | const auto& array_column = static_cast<const ColumnArray&>(*argument_column); | 389 | | const auto& offset_column_data = array_column.get_offsets(); | 390 | | const auto& nested_nullable_column = | 391 | | static_cast<const ColumnNullable&>(array_column.get_data()); | 392 | | const auto& nested_column = nested_nullable_column.get_nested_column(); | 393 | | const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data(); | 394 | | | 395 | | switch (argument_type->get_primitive_type()) { | 396 | | case PrimitiveType::TYPE_TINYINT: | 397 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column, | 398 | | nested_null_map, res, null_map)); | 399 | | break; | 400 | | case PrimitiveType::TYPE_BOOLEAN: | 401 | | RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>( | 402 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 403 | | break; | 404 | | case PrimitiveType::TYPE_SMALLINT: | 405 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt16>( | 406 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 407 | | break; | 408 | | case PrimitiveType::TYPE_INT: | 409 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt32>( | 410 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 411 | | break; | 412 | | case PrimitiveType::TYPE_BIGINT: | 413 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt64>( | 414 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 415 | | break; | 416 | | default: | 417 | | return Status::RuntimeError("Illegal column {} of argument of function {}", | 418 | | block.get_by_position(arguments[0]).column->get_name(), | 419 | | get_name()); | 420 | | } | 421 | | } else { | 422 | | return Status::RuntimeError("Illegal column {} of argument of function {}", | 423 | | block.get_by_position(arguments[0]).column->get_name(), | 424 | | get_name()); | 425 | | } | 426 | 281 | block.get_by_position(result).column = | 427 | 281 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 428 | 281 | return Status::OK(); | 429 | 281 | } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 372 | 25 | uint32_t result, size_t input_rows_count) const override { | 373 | 25 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 374 | 25 | auto res_data_column = ColumnBitmap::create(); | 375 | 25 | auto& null_map = res_null_map->get_data(); | 376 | 25 | auto& res = res_data_column->get_data(); | 377 | | | 378 | 25 | ColumnPtr& argument_column = block.get_by_position(arguments[0]).column; | 379 | 25 | if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) { | 380 | 25 | const auto& str_column = static_cast<const ColumnString&>(*argument_column); | 381 | 25 | const ColumnString::Chars& data = str_column.get_chars(); | 382 | 25 | const ColumnString::Offsets& offsets = str_column.get_offsets(); | 383 | 25 | RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count)); | 384 | | } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) { | 385 | | auto argument_type = remove_nullable( | 386 | | assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type) | 387 | | .get_nested_type()); | 388 | | const auto& array_column = static_cast<const ColumnArray&>(*argument_column); | 389 | | const auto& offset_column_data = array_column.get_offsets(); | 390 | | const auto& nested_nullable_column = | 391 | | static_cast<const ColumnNullable&>(array_column.get_data()); | 392 | | const auto& nested_column = nested_nullable_column.get_nested_column(); | 393 | | const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data(); | 394 | | | 395 | | switch (argument_type->get_primitive_type()) { | 396 | | case PrimitiveType::TYPE_TINYINT: | 397 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column, | 398 | | nested_null_map, res, null_map)); | 399 | | break; | 400 | | case PrimitiveType::TYPE_BOOLEAN: | 401 | | RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>( | 402 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 403 | | break; | 404 | | case PrimitiveType::TYPE_SMALLINT: | 405 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt16>( | 406 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 407 | | break; | 408 | | case PrimitiveType::TYPE_INT: | 409 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt32>( | 410 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 411 | | break; | 412 | | case PrimitiveType::TYPE_BIGINT: | 413 | | RETURN_IF_ERROR(Impl::template vector<ColumnInt64>( | 414 | | offset_column_data, nested_column, nested_null_map, res, null_map)); | 415 | | break; | 416 | | default: | 417 | | return Status::RuntimeError("Illegal column {} of argument of function {}", | 418 | | block.get_by_position(arguments[0]).column->get_name(), | 419 | | get_name()); | 420 | | } | 421 | | } else { | 422 | | return Status::RuntimeError("Illegal column {} of argument of function {}", | 423 | | block.get_by_position(arguments[0]).column->get_name(), | 424 | | get_name()); | 425 | | } | 426 | 24 | block.get_by_position(result).column = | 427 | 25 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 428 | 25 | return Status::OK(); | 429 | 25 | } |
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 372 | 13 | uint32_t result, size_t input_rows_count) const override { | 373 | 13 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 374 | 13 | auto res_data_column = ColumnBitmap::create(); | 375 | 13 | auto& null_map = res_null_map->get_data(); | 376 | 13 | auto& res = res_data_column->get_data(); | 377 | | | 378 | 13 | ColumnPtr& argument_column = block.get_by_position(arguments[0]).column; | 379 | | if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) { | 380 | | const auto& str_column = static_cast<const ColumnString&>(*argument_column); | 381 | | const ColumnString::Chars& data = str_column.get_chars(); | 382 | | const ColumnString::Offsets& offsets = str_column.get_offsets(); | 383 | | RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count)); | 384 | 13 | } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) { | 385 | 13 | auto argument_type = remove_nullable( | 386 | 13 | assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type) | 387 | 13 | .get_nested_type()); | 388 | 13 | const auto& array_column = static_cast<const ColumnArray&>(*argument_column); | 389 | 13 | const auto& offset_column_data = array_column.get_offsets(); | 390 | 13 | const auto& nested_nullable_column = | 391 | 13 | static_cast<const ColumnNullable&>(array_column.get_data()); | 392 | 13 | const auto& nested_column = nested_nullable_column.get_nested_column(); | 393 | 13 | const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data(); | 394 | | | 395 | 13 | switch (argument_type->get_primitive_type()) { | 396 | 3 | case PrimitiveType::TYPE_TINYINT: | 397 | 3 | RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column, | 398 | 3 | nested_null_map, res, null_map)); | 399 | 3 | break; | 400 | 3 | case PrimitiveType::TYPE_BOOLEAN: | 401 | 0 | RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>( | 402 | 0 | offset_column_data, nested_column, nested_null_map, res, null_map)); | 403 | 0 | break; | 404 | 0 | case PrimitiveType::TYPE_SMALLINT: | 405 | 0 | RETURN_IF_ERROR(Impl::template vector<ColumnInt16>( | 406 | 0 | offset_column_data, nested_column, nested_null_map, res, null_map)); | 407 | 0 | break; | 408 | 3 | case PrimitiveType::TYPE_INT: | 409 | 3 | RETURN_IF_ERROR(Impl::template vector<ColumnInt32>( | 410 | 3 | offset_column_data, nested_column, nested_null_map, res, null_map)); | 411 | 3 | break; | 412 | 7 | case PrimitiveType::TYPE_BIGINT: | 413 | 7 | RETURN_IF_ERROR(Impl::template vector<ColumnInt64>( | 414 | 7 | offset_column_data, nested_column, nested_null_map, res, null_map)); | 415 | 7 | break; | 416 | 7 | default: | 417 | 0 | return Status::RuntimeError("Illegal column {} of argument of function {}", | 418 | 0 | block.get_by_position(arguments[0]).column->get_name(), | 419 | 0 | get_name()); | 420 | 13 | } | 421 | | } else { | 422 | | return Status::RuntimeError("Illegal column {} of argument of function {}", | 423 | | block.get_by_position(arguments[0]).column->get_name(), | 424 | | get_name()); | 425 | | } | 426 | 13 | block.get_by_position(result).column = | 427 | 13 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 428 | 13 | return Status::OK(); | 429 | 13 | } |
|
430 | | }; |
431 | | |
432 | | template <int HashBits> |
433 | | struct BitmapHashName {}; |
434 | | |
435 | | template <> |
436 | | struct BitmapHashName<32> { |
437 | | static constexpr auto name = "bitmap_hash"; |
438 | | }; |
439 | | |
440 | | template <> |
441 | | struct BitmapHashName<64> { |
442 | | static constexpr auto name = "bitmap_hash64"; |
443 | | }; |
444 | | |
445 | | template <int HashBits> |
446 | | struct BitmapHash { |
447 | | static constexpr auto name = BitmapHashName<HashBits>::name; |
448 | | |
449 | | using ReturnType = DataTypeBitMap; |
450 | | |
451 | | template <typename ColumnType> |
452 | 130 | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { |
453 | 130 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { |
454 | 130 | const ColumnString::Chars& data = col->get_chars(); |
455 | 130 | const ColumnString::Offsets& offsets = col->get_offsets(); |
456 | 130 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
457 | 130 | auto& res_data = res_column->get_data(); |
458 | 130 | size_t size = offsets.size(); |
459 | | |
460 | 436 | for (size_t i = 0; i < size; ++i) { |
461 | 306 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
462 | 306 | size_t str_size = offsets[i] - offsets[i - 1]; |
463 | 306 | if constexpr (HashBits == 32) { |
464 | 274 | uint32_t hash_value = |
465 | 274 | HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED); |
466 | 274 | res_data[i].add(hash_value); |
467 | 274 | } else { |
468 | 32 | uint64_t hash_value = 0; |
469 | 32 | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); |
470 | 32 | res_data[i].add(hash_value); |
471 | 32 | } |
472 | 306 | } |
473 | 130 | } |
474 | 130 | } _ZN5doris10BitmapHashILi32EE6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE Line | Count | Source | 452 | 114 | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { | 453 | 114 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 454 | 114 | const ColumnString::Chars& data = col->get_chars(); | 455 | 114 | const ColumnString::Offsets& offsets = col->get_offsets(); | 456 | 114 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 457 | 114 | auto& res_data = res_column->get_data(); | 458 | 114 | size_t size = offsets.size(); | 459 | | | 460 | 388 | for (size_t i = 0; i < size; ++i) { | 461 | 274 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 462 | 274 | size_t str_size = offsets[i] - offsets[i - 1]; | 463 | 274 | if constexpr (HashBits == 32) { | 464 | 274 | uint32_t hash_value = | 465 | 274 | HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED); | 466 | 274 | res_data[i].add(hash_value); | 467 | | } else { | 468 | | uint64_t hash_value = 0; | 469 | | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); | 470 | | res_data[i].add(hash_value); | 471 | | } | 472 | 274 | } | 473 | 114 | } | 474 | 114 | } |
Unexecuted instantiation: _ZN5doris10BitmapHashILi32EE6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE _ZN5doris10BitmapHashILi64EE6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE Line | Count | Source | 452 | 16 | static void vector(const ColumnType* col, MutableColumnPtr& col_res) { | 453 | 16 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 454 | 16 | const ColumnString::Chars& data = col->get_chars(); | 455 | 16 | const ColumnString::Offsets& offsets = col->get_offsets(); | 456 | 16 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 457 | 16 | auto& res_data = res_column->get_data(); | 458 | 16 | size_t size = offsets.size(); | 459 | | | 460 | 48 | for (size_t i = 0; i < size; ++i) { | 461 | 32 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 462 | 32 | size_t str_size = offsets[i] - offsets[i - 1]; | 463 | | if constexpr (HashBits == 32) { | 464 | | uint32_t hash_value = | 465 | | HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED); | 466 | | res_data[i].add(hash_value); | 467 | 32 | } else { | 468 | 32 | uint64_t hash_value = 0; | 469 | 32 | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); | 470 | 32 | res_data[i].add(hash_value); | 471 | 32 | } | 472 | 32 | } | 473 | 16 | } | 474 | 16 | } |
Unexecuted instantiation: _ZN5doris10BitmapHashILi64EE6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE |
475 | | |
476 | | template <typename ColumnType> |
477 | | static void vector_nullable(const ColumnType* col, const NullMap& nullmap, |
478 | 192 | MutableColumnPtr& col_res) { |
479 | 192 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { |
480 | 192 | const ColumnString::Chars& data = col->get_chars(); |
481 | 192 | const ColumnString::Offsets& offsets = col->get_offsets(); |
482 | 192 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); |
483 | 192 | auto& res_data = res_column->get_data(); |
484 | 192 | size_t size = offsets.size(); |
485 | | |
486 | 991 | for (size_t i = 0; i < size; ++i) { |
487 | 799 | if (nullmap[i]) { |
488 | 57 | continue; |
489 | 742 | } else { |
490 | 742 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); |
491 | 742 | size_t str_size = offsets[i] - offsets[i - 1]; |
492 | 742 | if constexpr (HashBits == 32) { |
493 | 716 | uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size, |
494 | 716 | HashUtil::MURMUR3_32_SEED); |
495 | 716 | res_data[i].add(hash_value); |
496 | 716 | } else { |
497 | 26 | uint64_t hash_value = 0; |
498 | 26 | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); |
499 | 26 | res_data[i].add(hash_value); |
500 | 26 | } |
501 | 742 | } |
502 | 799 | } |
503 | 192 | } |
504 | 192 | } _ZN5doris10BitmapHashILi32EE15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE Line | Count | Source | 478 | 183 | MutableColumnPtr& col_res) { | 479 | 183 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 480 | 183 | const ColumnString::Chars& data = col->get_chars(); | 481 | 183 | const ColumnString::Offsets& offsets = col->get_offsets(); | 482 | 183 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 483 | 183 | auto& res_data = res_column->get_data(); | 484 | 183 | size_t size = offsets.size(); | 485 | | | 486 | 955 | for (size_t i = 0; i < size; ++i) { | 487 | 772 | if (nullmap[i]) { | 488 | 56 | continue; | 489 | 716 | } else { | 490 | 716 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 491 | 716 | size_t str_size = offsets[i] - offsets[i - 1]; | 492 | 716 | if constexpr (HashBits == 32) { | 493 | 716 | uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size, | 494 | 716 | HashUtil::MURMUR3_32_SEED); | 495 | 716 | res_data[i].add(hash_value); | 496 | | } else { | 497 | | uint64_t hash_value = 0; | 498 | | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); | 499 | | res_data[i].add(hash_value); | 500 | | } | 501 | 716 | } | 502 | 772 | } | 503 | 183 | } | 504 | 183 | } |
Unexecuted instantiation: _ZN5doris10BitmapHashILi32EE15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE _ZN5doris10BitmapHashILi64EE15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE Line | Count | Source | 478 | 9 | MutableColumnPtr& col_res) { | 479 | 9 | if constexpr (std::is_same_v<ColumnType, ColumnString>) { | 480 | 9 | const ColumnString::Chars& data = col->get_chars(); | 481 | 9 | const ColumnString::Offsets& offsets = col->get_offsets(); | 482 | 9 | auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get()); | 483 | 9 | auto& res_data = res_column->get_data(); | 484 | 9 | size_t size = offsets.size(); | 485 | | | 486 | 36 | for (size_t i = 0; i < size; ++i) { | 487 | 27 | if (nullmap[i]) { | 488 | 1 | continue; | 489 | 26 | } else { | 490 | 26 | const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]); | 491 | 26 | size_t str_size = offsets[i] - offsets[i - 1]; | 492 | | if constexpr (HashBits == 32) { | 493 | | uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size, | 494 | | HashUtil::MURMUR3_32_SEED); | 495 | | res_data[i].add(hash_value); | 496 | 26 | } else { | 497 | 26 | uint64_t hash_value = 0; | 498 | 26 | murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value); | 499 | 26 | res_data[i].add(hash_value); | 500 | 26 | } | 501 | 26 | } | 502 | 27 | } | 503 | 9 | } | 504 | 9 | } |
Unexecuted instantiation: _ZN5doris10BitmapHashILi64EE15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE |
505 | | }; |
506 | | |
507 | | class FunctionBitmapCount : public IFunction { |
508 | | public: |
509 | | static constexpr auto name = "bitmap_count"; |
510 | | |
511 | 1 | String get_name() const override { return name; } |
512 | | |
513 | 80 | static FunctionPtr create() { return std::make_shared<FunctionBitmapCount>(); } |
514 | | |
515 | 71 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { |
516 | 71 | return std::make_shared<DataTypeInt64>(); |
517 | 71 | } |
518 | | |
519 | 71 | size_t get_number_of_arguments() const override { return 1; } |
520 | | |
521 | 188 | bool use_default_implementation_for_nulls() const override { return false; } |
522 | | |
523 | | Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
524 | 116 | uint32_t result, size_t input_rows_count) const override { |
525 | 116 | auto res_data_column = ColumnInt64::create(); |
526 | 116 | auto& res = res_data_column->get_data(); |
527 | 116 | auto data_null_map = ColumnUInt8::create(input_rows_count, 0); |
528 | 116 | auto& null_map = data_null_map->get_data(); |
529 | | |
530 | 116 | auto column = block.get_by_position(arguments[0]).column; |
531 | 116 | if (auto* nullable = check_and_get_column<const ColumnNullable>(*column)) { |
532 | 19 | VectorizedUtils::update_null_map(null_map, nullable->get_null_map_data()); |
533 | 19 | column = nullable->get_nested_column_ptr(); |
534 | 19 | } |
535 | 116 | auto str_col = assert_cast<const ColumnBitmap*>(column.get()); |
536 | 116 | const auto& col_data = str_col->get_data(); |
537 | | |
538 | 116 | res.reserve(input_rows_count); |
539 | 321 | for (size_t i = 0; i < input_rows_count; ++i) { |
540 | 205 | if (null_map[i]) { |
541 | 3 | res.push_back(0); |
542 | 3 | continue; |
543 | 3 | } |
544 | 202 | res.push_back(col_data[i].cardinality()); |
545 | 202 | } |
546 | 116 | block.replace_by_position(result, std::move(res_data_column)); |
547 | 116 | return Status::OK(); |
548 | 116 | } |
549 | | }; |
550 | | |
551 | | struct NameBitmapNot { |
552 | | static constexpr auto name = "bitmap_not"; |
553 | | }; |
554 | | |
555 | | template <typename LeftDataType, typename RightDataType> |
556 | | struct BitmapNot { |
557 | | using ResultDataType = DataTypeBitMap; |
558 | | using T0 = typename LeftDataType::FieldType; |
559 | | using T1 = typename RightDataType::FieldType; |
560 | | using TData = std::vector<BitmapValue>; |
561 | | |
562 | 10 | static void vector_vector(const TData& lvec, const TData& rvec, TData& res) { |
563 | 10 | size_t size = lvec.size(); |
564 | 37 | for (size_t i = 0; i < size; ++i) { |
565 | 27 | res[i] = lvec[i]; |
566 | 27 | res[i] -= rvec[i]; |
567 | 27 | } |
568 | 10 | } |
569 | 0 | static void vector_scalar(const TData& lvec, const BitmapValue& rval, TData& res) { |
570 | 0 | size_t size = lvec.size(); |
571 | 0 | for (size_t i = 0; i < size; ++i) { |
572 | 0 | res[i] = lvec[i]; |
573 | 0 | res[i] -= rval; |
574 | 0 | } |
575 | 0 | } |
576 | 0 | static void scalar_vector(const BitmapValue& lval, const TData& rvec, TData& res) { |
577 | 0 | size_t size = rvec.size(); |
578 | 0 | for (size_t i = 0; i < size; ++i) { |
579 | 0 | res[i] = lval; |
580 | 0 | res[i] -= rvec[i]; |
581 | 0 | } |
582 | 0 | } |
583 | | }; |
584 | | |
585 | | struct NameBitmapAndNot { |
586 | | static constexpr auto name = "bitmap_and_not"; |
587 | | }; |
588 | | |
589 | | template <typename LeftDataType, typename RightDataType> |
590 | | struct BitmapAndNot { |
591 | | using ResultDataType = DataTypeBitMap; |
592 | | using T0 = typename LeftDataType::FieldType; |
593 | | using T1 = typename RightDataType::FieldType; |
594 | | using TData = std::vector<BitmapValue>; |
595 | | |
596 | 19 | static void vector_vector(const TData& lvec, const TData& rvec, TData& res) { |
597 | 19 | size_t size = lvec.size(); |
598 | 19 | BitmapValue mid_data; |
599 | 74 | for (size_t i = 0; i < size; ++i) { |
600 | 55 | mid_data = lvec[i]; |
601 | 55 | mid_data &= rvec[i]; |
602 | 55 | res[i] = lvec[i]; |
603 | 55 | res[i] -= mid_data; |
604 | 55 | mid_data.reset(); |
605 | 55 | } |
606 | 19 | } |
607 | 0 | static void vector_scalar(const TData& lvec, const BitmapValue& rval, TData& res) { |
608 | 0 | size_t size = lvec.size(); |
609 | 0 | BitmapValue mid_data; |
610 | 0 | for (size_t i = 0; i < size; ++i) { |
611 | 0 | mid_data = lvec[i]; |
612 | 0 | mid_data &= rval; |
613 | 0 | res[i] = lvec[i]; |
614 | 0 | res[i] -= mid_data; |
615 | 0 | mid_data.reset(); |
616 | 0 | } |
617 | 0 | } |
618 | 0 | static void scalar_vector(const BitmapValue& lval, const TData& rvec, TData& res) { |
619 | 0 | size_t size = rvec.size(); |
620 | 0 | BitmapValue mid_data; |
621 | 0 | for (size_t i = 0; i < size; ++i) { |
622 | 0 | mid_data = lval; |
623 | 0 | mid_data &= rvec[i]; |
624 | 0 | res[i] = lval; |
625 | 0 | res[i] -= mid_data; |
626 | 0 | mid_data.reset(); |
627 | 0 | } |
628 | 0 | } |
629 | | }; |
630 | | |
631 | | struct NameBitmapAndNotCount { |
632 | | static constexpr auto name = "bitmap_and_not_count"; |
633 | | }; |
634 | | |
635 | | template <typename LeftDataType, typename RightDataType> |
636 | | struct BitmapAndNotCount { |
637 | | using ResultDataType = DataTypeInt64; |
638 | | using T0 = typename LeftDataType::FieldType; |
639 | | using T1 = typename RightDataType::FieldType; |
640 | | using TData = std::vector<BitmapValue>; |
641 | | using ResTData = typename ColumnInt64::Container::value_type; |
642 | | |
643 | 29 | static void vector_vector(const TData& lvec, const TData& rvec, ResTData* res) { |
644 | 29 | size_t size = lvec.size(); |
645 | 29 | BitmapValue mid_data; |
646 | 107 | for (size_t i = 0; i < size; ++i) { |
647 | 78 | mid_data = lvec[i]; |
648 | 78 | mid_data &= rvec[i]; |
649 | 78 | res[i] = lvec[i].andnot_cardinality(mid_data); |
650 | 78 | mid_data.reset(); |
651 | 78 | } |
652 | 29 | } |
653 | 0 | static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData* res) { |
654 | 0 | size_t size = rvec.size(); |
655 | 0 | BitmapValue mid_data; |
656 | 0 | for (size_t i = 0; i < size; ++i) { |
657 | 0 | mid_data = lval; |
658 | 0 | mid_data &= rvec[i]; |
659 | 0 | res[i] = lval.andnot_cardinality(mid_data); |
660 | 0 | mid_data.reset(); |
661 | 0 | } |
662 | 0 | } |
663 | 0 | static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData* res) { |
664 | 0 | size_t size = lvec.size(); |
665 | 0 | BitmapValue mid_data; |
666 | 0 | for (size_t i = 0; i < size; ++i) { |
667 | 0 | mid_data = lvec[i]; |
668 | 0 | mid_data &= rval; |
669 | 0 | res[i] = lvec[i].andnot_cardinality(mid_data); |
670 | 0 | mid_data.reset(); |
671 | 0 | } |
672 | 0 | } |
673 | | }; |
674 | | |
675 | 63 | void update_bitmap_op_count(int64_t* __restrict count, const NullMap& null_map) { |
676 | 63 | static constexpr int64_t flags[2] = {-1, 0}; |
677 | 63 | size_t size = null_map.size(); |
678 | 63 | auto* __restrict null_map_data = null_map.data(); |
679 | 178 | for (size_t i = 0; i < size; ++i) { |
680 | 115 | count[i] &= flags[null_map_data[i]]; |
681 | 115 | } |
682 | 63 | } |
683 | | |
684 | | // for bitmap_and_count, bitmap_xor_count and bitmap_and_not_count, |
685 | | // result is 0 for rows that if any column is null value |
686 | | ColumnPtr handle_bitmap_op_count_null_value(ColumnPtr& src, const Block& block, |
687 | | const ColumnNumbers& args, uint32_t result, |
688 | 34 | size_t input_rows_count) { |
689 | 34 | MutableColumnPtr mutable_src = IColumn::mutate(std::move(src)); |
690 | 34 | auto* nullable = assert_cast<ColumnNullable*>(mutable_src.get()); |
691 | 34 | auto* src_not_nullable_mutable = &nullable->get_nested_column(); |
692 | 34 | auto* __restrict count_data = |
693 | 34 | assert_cast<ColumnInt64*>(src_not_nullable_mutable)->get_data().data(); |
694 | | |
695 | 80 | for (const auto& arg : args) { |
696 | 80 | const ColumnWithTypeAndName& elem = block.get_by_position(arg); |
697 | 80 | if (!elem.type->is_nullable()) { |
698 | 17 | continue; |
699 | 17 | } |
700 | | |
701 | 63 | bool is_const = is_column_const(*elem.column); |
702 | | /// Const Nullable that are NULL. |
703 | 63 | if (is_const && assert_cast<const ColumnConst*>(elem.column.get())->only_null()) { |
704 | 0 | return block.get_by_position(result).type->create_column_const( |
705 | 0 | input_rows_count, Field::create_field<TYPE_BIGINT>(0)); |
706 | 0 | } |
707 | 63 | if (is_const) { |
708 | 0 | continue; |
709 | 0 | } |
710 | | |
711 | 63 | if (const auto* nullable_column = assert_cast<const ColumnNullable*>(elem.column.get())) { |
712 | 63 | const ColumnPtr& null_map_column = nullable_column->get_null_map_column_ptr(); |
713 | 63 | const NullMap& src_null_map = |
714 | 63 | assert_cast<const ColumnUInt8&>(*null_map_column).get_data(); |
715 | | |
716 | 63 | update_bitmap_op_count(count_data, src_null_map); |
717 | 63 | } |
718 | 63 | } |
719 | | |
720 | 34 | return mutable_src; |
721 | 34 | } |
722 | | |
723 | | Status execute_bitmap_op_count_null_to_zero( |
724 | | FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, |
725 | | size_t input_rows_count, |
726 | | const std::function<Status(FunctionContext*, Block&, const ColumnNumbers&, size_t, size_t)>& |
727 | 75 | exec_impl_func) { |
728 | 75 | if (have_null_column(block, arguments)) { |
729 | 34 | auto [temporary_block, new_args, new_result] = |
730 | 34 | create_block_with_nested_columns(block, arguments, result); |
731 | 34 | RETURN_IF_ERROR(exec_impl_func(context, temporary_block, new_args, new_result, |
732 | 34 | temporary_block.rows())); |
733 | 34 | block.get_by_position(result).column = handle_bitmap_op_count_null_value( |
734 | 34 | temporary_block.get_by_position(new_result).column, block, arguments, result, |
735 | 34 | input_rows_count); |
736 | 41 | } else { |
737 | 41 | return exec_impl_func(context, block, arguments, result, input_rows_count); |
738 | 41 | } |
739 | 34 | return Status::OK(); |
740 | 75 | } |
741 | | |
742 | | template <typename FunctionName> |
743 | | class FunctionBitmapAndNotCount : public IFunction { |
744 | | public: |
745 | | using LeftDataType = DataTypeBitMap; |
746 | | using RightDataType = DataTypeBitMap; |
747 | | using ResultDataType = typename BitmapAndNotCount<LeftDataType, RightDataType>::ResultDataType; |
748 | | |
749 | | static constexpr auto name = FunctionName::name; |
750 | 23 | static FunctionPtr create() { return std::make_shared<FunctionBitmapAndNotCount>(); } |
751 | 1 | String get_name() const override { return name; } |
752 | 14 | size_t get_number_of_arguments() const override { return 2; } |
753 | 14 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { |
754 | 14 | bool return_nullable = false; |
755 | | // result is nullable only when any columns is nullable for bitmap_and_not_count |
756 | 31 | for (size_t i = 0; i < arguments.size(); ++i) { |
757 | 24 | if (arguments[i]->is_nullable()) { |
758 | 7 | return_nullable = true; |
759 | 7 | break; |
760 | 7 | } |
761 | 24 | } |
762 | 14 | auto result_type = std::make_shared<ResultDataType>(); |
763 | 14 | return return_nullable ? make_nullable(result_type) : result_type; |
764 | 14 | } |
765 | | |
766 | 43 | bool use_default_implementation_for_nulls() const override { |
767 | | // for bitmap_and_not_count, result is always not null, and if the bitmap op result is null, |
768 | | // the count is 0 |
769 | 43 | return false; |
770 | 43 | } |
771 | | |
772 | | Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
773 | 29 | uint32_t result, size_t input_rows_count) const override { |
774 | 29 | DCHECK_EQ(arguments.size(), 2); |
775 | 29 | auto impl_func = [&](FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
776 | 29 | uint32_t result, size_t input_rows_count) { |
777 | 29 | return execute_impl_internal(context, block, arguments, result, input_rows_count); |
778 | 29 | }; |
779 | 29 | return execute_bitmap_op_count_null_to_zero(context, block, arguments, result, |
780 | 29 | input_rows_count, impl_func); |
781 | 29 | } |
782 | | |
783 | | Status execute_impl_internal(FunctionContext* context, Block& block, |
784 | | const ColumnNumbers& arguments, uint32_t result, |
785 | 29 | size_t input_rows_count) const { |
786 | 29 | using ColVecResult = ColumnVector<ResultDataType::PType>; |
787 | | |
788 | 29 | typename ColVecResult::MutablePtr col_res = ColVecResult::create(); |
789 | 29 | auto& vec_res = col_res->get_data(); |
790 | 29 | vec_res.resize(block.rows()); |
791 | | |
792 | 29 | const auto& left = block.get_by_position(arguments[0]); |
793 | 29 | auto lcol = left.column; |
794 | 29 | const auto& right = block.get_by_position(arguments[1]); |
795 | 29 | auto rcol = right.column; |
796 | | |
797 | 29 | if (is_column_const(*left.column)) { |
798 | 0 | BitmapAndNotCount<LeftDataType, RightDataType>::scalar_vector( |
799 | 0 | assert_cast<const ColumnBitmap&>( |
800 | 0 | assert_cast<const ColumnConst*>(lcol.get())->get_data_column()) |
801 | 0 | .get_data()[0], |
802 | 0 | assert_cast<const ColumnBitmap*>(rcol.get())->get_data(), vec_res.data()); |
803 | 29 | } else if (is_column_const(*right.column)) { |
804 | 0 | BitmapAndNotCount<LeftDataType, RightDataType>::vector_scalar( |
805 | 0 | assert_cast<const ColumnBitmap*>(lcol.get())->get_data(), |
806 | 0 | assert_cast<const ColumnBitmap&>( |
807 | 0 | assert_cast<const ColumnConst*>(rcol.get())->get_data_column()) |
808 | 0 | .get_data()[0], |
809 | 0 | vec_res.data()); |
810 | 29 | } else { |
811 | 29 | BitmapAndNotCount<LeftDataType, RightDataType>::vector_vector( |
812 | 29 | assert_cast<const ColumnBitmap*>(lcol.get())->get_data(), |
813 | 29 | assert_cast<const ColumnBitmap*>(rcol.get())->get_data(), vec_res.data()); |
814 | 29 | } |
815 | | |
816 | 29 | auto& result_info = block.get_by_position(result); |
817 | 29 | if (result_info.type->is_nullable()) { |
818 | 10 | block.replace_by_position( |
819 | 10 | result, ColumnNullable::create(std::move(col_res), |
820 | 10 | ColumnUInt8::create(input_rows_count, 0))); |
821 | 19 | } else { |
822 | 19 | block.replace_by_position(result, std::move(col_res)); |
823 | 19 | } |
824 | 29 | return Status::OK(); |
825 | 29 | } |
826 | | }; |
827 | | |
828 | | struct NameBitmapContains { |
829 | | static constexpr auto name = "bitmap_contains"; |
830 | | }; |
831 | | |
832 | | template <typename LeftDataType, typename RightDataType> |
833 | | struct BitmapContains { |
834 | | using ResultDataType = DataTypeUInt8; |
835 | | using T0 = typename LeftDataType::FieldType; |
836 | | using T1 = typename RightDataType::FieldType; |
837 | | using LTData = std::vector<BitmapValue>; |
838 | | using RTData = typename ColumnVector<RightDataType::PType>::Container; |
839 | | using ResTData = typename ColumnUInt8::Container; |
840 | | |
841 | 10 | static void vector_vector(const LTData& lvec, const RTData& rvec, ResTData& res) { |
842 | 10 | size_t size = lvec.size(); |
843 | 37 | for (size_t i = 0; i < size; ++i) { |
844 | 27 | res[i] = lvec[i].contains(rvec[i]); |
845 | 27 | } |
846 | 10 | } |
847 | 410 | static void vector_scalar(const LTData& lvec, const T1& rval, ResTData& res) { |
848 | 410 | size_t size = lvec.size(); |
849 | 820 | for (size_t i = 0; i < size; ++i) { |
850 | 410 | res[i] = lvec[i].contains(rval); |
851 | 410 | } |
852 | 410 | } |
853 | 0 | static void scalar_vector(const BitmapValue& lval, const RTData& rvec, ResTData& res) { |
854 | 0 | size_t size = rvec.size(); |
855 | 0 | for (size_t i = 0; i < size; ++i) { |
856 | 0 | res[i] = lval.contains(rvec[i]); |
857 | 0 | } |
858 | 0 | } |
859 | | }; |
860 | | |
861 | | struct NameBitmapRemove { |
862 | | static constexpr auto name = "bitmap_remove"; |
863 | | }; |
864 | | |
865 | | template <typename LeftDataType, typename RightDataType> |
866 | | struct BitmapRemove { |
867 | | using ResultDataType = DataTypeBitMap; |
868 | | using T0 = typename LeftDataType::FieldType; |
869 | | using T1 = typename RightDataType::FieldType; |
870 | | using LTData = std::vector<BitmapValue>; |
871 | | using RTData = typename ColumnVector<RightDataType::PType>::Container; |
872 | | using ResTData = std::vector<BitmapValue>; |
873 | | |
874 | 2 | static void vector_vector(const LTData& lvec, const RTData& rvec, ResTData& res) { |
875 | 2 | size_t size = lvec.size(); |
876 | 6 | for (size_t i = 0; i < size; ++i) { |
877 | 4 | res[i] = lvec[i]; |
878 | 4 | res[i].remove(rvec[i]); |
879 | 4 | } |
880 | 2 | } |
881 | 12 | static void vector_scalar(const LTData& lvec, const T1& rval, ResTData& res) { |
882 | 12 | size_t size = lvec.size(); |
883 | 45 | for (size_t i = 0; i < size; ++i) { |
884 | 33 | res[i] = lvec[i]; |
885 | 33 | res[i].remove(rval); |
886 | 33 | } |
887 | 12 | } |
888 | 0 | static void scalar_vector(const BitmapValue& lval, const RTData& rvec, ResTData& res) { |
889 | 0 | size_t size = rvec.size(); |
890 | 0 | for (size_t i = 0; i < size; ++i) { |
891 | 0 | res[i] = lval; |
892 | 0 | res[i].remove(rvec[i]); |
893 | 0 | } |
894 | 0 | } |
895 | | }; |
896 | | |
897 | | struct NameBitmapHasAny { |
898 | | static constexpr auto name = "bitmap_has_any"; |
899 | | }; |
900 | | |
901 | | template <typename LeftDataType, typename RightDataType> |
902 | | struct BitmapHasAny { |
903 | | using ResultDataType = DataTypeUInt8; |
904 | | using T0 = typename LeftDataType::FieldType; |
905 | | using T1 = typename RightDataType::FieldType; |
906 | | using TData = std::vector<BitmapValue>; |
907 | | using ResTData = typename ColumnUInt8::Container; |
908 | | |
909 | 0 | static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) { |
910 | 0 | size_t size = lvec.size(); |
911 | 0 | for (size_t i = 0; i < size; ++i) { |
912 | 0 | res[i] = lvec[i].intersects(rvec[i]); |
913 | 0 | } |
914 | 0 | } |
915 | 0 | static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) { |
916 | 0 | size_t size = lvec.size(); |
917 | 0 | for (size_t i = 0; i < size; ++i) { |
918 | 0 | res[i] = lvec[i].intersects(rval); |
919 | 0 | } |
920 | 0 | } |
921 | 0 | static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) { |
922 | 0 | size_t size = rvec.size(); |
923 | 0 | for (size_t i = 0; i < size; ++i) { |
924 | 0 | res[i] = lval.intersects(rvec[i]); |
925 | 0 | } |
926 | 0 | } |
927 | | }; |
928 | | |
929 | | struct NameBitmapHasAll { |
930 | | static constexpr auto name = "bitmap_has_all"; |
931 | | }; |
932 | | |
933 | | template <typename LeftDataType, typename RightDataType> |
934 | | struct BitmapHasAll { |
935 | | using ResultDataType = DataTypeUInt8; |
936 | | using T0 = typename LeftDataType::FieldType; |
937 | | using T1 = typename RightDataType::FieldType; |
938 | | using TData = std::vector<BitmapValue>; |
939 | | using ResTData = typename ColumnUInt8::Container; |
940 | | |
941 | 1 | static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) { |
942 | 1 | size_t size = lvec.size(); |
943 | 6 | for (size_t i = 0; i < size; ++i) { |
944 | 5 | res[i] = lvec[i].contains_all(rvec[i]); |
945 | 5 | } |
946 | 1 | } |
947 | 0 | static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) { |
948 | 0 | size_t size = lvec.size(); |
949 | 0 | for (size_t i = 0; i < size; ++i) { |
950 | 0 | res[i] = lvec[i].contains_all(rval); |
951 | 0 | } |
952 | 0 | } |
953 | 0 | static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) { |
954 | 0 | size_t size = rvec.size(); |
955 | 0 | for (size_t i = 0; i < size; ++i) { |
956 | 0 | res[i] = lval.contains_all(rvec[i]); |
957 | 0 | } |
958 | 0 | } |
959 | | }; |
960 | | |
961 | | struct NameBitmapToString { |
962 | | static constexpr auto name = "bitmap_to_string"; |
963 | | }; |
964 | | |
965 | | struct BitmapToString { |
966 | | using ReturnType = DataTypeString; |
967 | | static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_BITMAP; |
968 | | using Type = DataTypeBitMap::FieldType; |
969 | | using ReturnColumnType = ColumnString; |
970 | | using Chars = ColumnString::Chars; |
971 | | using Offsets = ColumnString::Offsets; |
972 | | |
973 | 236 | static Status vector(const std::vector<BitmapValue>& data, Chars& chars, Offsets& offsets) { |
974 | 236 | size_t size = data.size(); |
975 | 236 | offsets.resize(size); |
976 | 236 | chars.reserve(size); |
977 | 1.20k | for (size_t i = 0; i < size; ++i) { |
978 | 967 | StringOP::push_value_string(data[i].to_string(), i, chars, offsets); |
979 | 967 | } |
980 | 236 | return Status::OK(); |
981 | 236 | } |
982 | | }; |
983 | | |
984 | | struct NameBitmapToBase64 { |
985 | | static constexpr auto name = "bitmap_to_base64"; |
986 | | }; |
987 | | |
988 | | struct BitmapToBase64 { |
989 | | using ReturnType = DataTypeString; |
990 | | static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_BITMAP; |
991 | | using Type = DataTypeBitMap::FieldType; |
992 | | using ReturnColumnType = ColumnString; |
993 | | using Chars = ColumnString::Chars; |
994 | | using Offsets = ColumnString::Offsets; |
995 | | |
996 | | // ColumnString not support 64bit, only 32bit, so that the max size is 4G |
997 | 25 | static Status vector(const std::vector<BitmapValue>& data, Chars& chars, Offsets& offsets) { |
998 | 25 | size_t size = data.size(); |
999 | 25 | offsets.resize(size); |
1000 | 25 | size_t output_char_size = 0; |
1001 | 71 | for (size_t i = 0; i < size; ++i) { |
1002 | 46 | const BitmapValue& bitmap_val = data[i]; |
1003 | 46 | auto ser_size = bitmap_val.getSizeInBytes(); |
1004 | 46 | output_char_size += (int)(4.0 * ceil((double)ser_size / 3.0)); |
1005 | 46 | } |
1006 | 25 | ColumnString::check_chars_length(output_char_size, size); |
1007 | 25 | chars.resize(output_char_size); |
1008 | 25 | auto chars_data = chars.data(); |
1009 | | |
1010 | 25 | size_t cur_ser_size = 0; |
1011 | 25 | size_t last_ser_size = 0; |
1012 | 25 | std::string ser_buff; |
1013 | 25 | size_t encoded_offset = 0; |
1014 | 71 | for (size_t i = 0; i < size; ++i) { |
1015 | 46 | const BitmapValue& bitmap_val = data[i]; |
1016 | 46 | cur_ser_size = bitmap_val.getSizeInBytes(); |
1017 | 46 | if (cur_ser_size > last_ser_size) { |
1018 | 35 | last_ser_size = cur_ser_size; |
1019 | 35 | ser_buff.resize(cur_ser_size); |
1020 | 35 | } |
1021 | 46 | bitmap_val.write_to(ser_buff.data()); |
1022 | | |
1023 | 46 | auto outlen = base64_encode((const unsigned char*)ser_buff.data(), cur_ser_size, |
1024 | 46 | chars_data + encoded_offset); |
1025 | 46 | DCHECK(outlen > 0); |
1026 | | |
1027 | 46 | encoded_offset += (int)(4.0 * ceil((double)cur_ser_size / 3.0)); |
1028 | 46 | offsets[i] = cast_set<UInt32>(encoded_offset); |
1029 | 46 | } |
1030 | 25 | return Status::OK(); |
1031 | 25 | } |
1032 | | }; |
1033 | | |
1034 | | struct SubBitmap { |
1035 | | static constexpr auto name = "sub_bitmap"; |
1036 | | using TData1 = std::vector<BitmapValue>; |
1037 | | using TData2 = typename ColumnInt64::Container; |
1038 | | |
1039 | | static void vector3(const TData1& bitmap_data, const TData2& offset_data, |
1040 | | const TData2& limit_data, NullMap& null_map, size_t input_rows_count, |
1041 | 13 | TData1& res) { |
1042 | 43 | for (int i = 0; i < input_rows_count; ++i) { |
1043 | 30 | if (null_map[i]) { |
1044 | 0 | continue; |
1045 | 0 | } |
1046 | 30 | if (limit_data[i] <= 0) { |
1047 | 1 | null_map[i] = 1; |
1048 | 1 | continue; |
1049 | 1 | } |
1050 | 29 | if (bitmap_data[i].offset_limit(offset_data[i], limit_data[i], &res[i]) == 0) { |
1051 | 24 | null_map[i] = 1; |
1052 | 24 | } |
1053 | 29 | } |
1054 | 13 | } |
1055 | | static void vector_scalars(const TData1& bitmap_data, const Int64& offset_data, |
1056 | | const Int64& limit_data, NullMap& null_map, size_t input_rows_count, |
1057 | 0 | TData1& res) { |
1058 | 0 | for (int i = 0; i < input_rows_count; ++i) { |
1059 | 0 | if (null_map[i]) { |
1060 | 0 | continue; |
1061 | 0 | } |
1062 | 0 | if (limit_data <= 0) { |
1063 | 0 | null_map[i] = 1; |
1064 | 0 | continue; |
1065 | 0 | } |
1066 | 0 | if (bitmap_data[i].offset_limit(offset_data, limit_data, &res[i]) == 0) { |
1067 | 0 | null_map[i] = 1; |
1068 | 0 | } |
1069 | 0 | } |
1070 | 0 | } |
1071 | | }; |
1072 | | |
1073 | | struct BitmapSubsetLimit { |
1074 | | static constexpr auto name = "bitmap_subset_limit"; |
1075 | | using TData1 = std::vector<BitmapValue>; |
1076 | | using TData2 = typename ColumnInt64::Container; |
1077 | | |
1078 | | static void vector3(const TData1& bitmap_data, const TData2& offset_data, |
1079 | | const TData2& limit_data, NullMap& null_map, size_t input_rows_count, |
1080 | 11 | TData1& res) { |
1081 | 39 | for (int i = 0; i < input_rows_count; ++i) { |
1082 | 28 | if (null_map[i]) { |
1083 | 0 | continue; |
1084 | 0 | } |
1085 | 28 | if (offset_data[i] < 0 || limit_data[i] < 0) { |
1086 | 0 | null_map[i] = 1; |
1087 | 0 | continue; |
1088 | 0 | } |
1089 | 28 | bitmap_data[i].sub_limit(offset_data[i], limit_data[i], &res[i]); |
1090 | 28 | } |
1091 | 11 | } |
1092 | | static void vector_scalars(const TData1& bitmap_data, const Int64& offset_data, |
1093 | | const Int64& limit_data, NullMap& null_map, size_t input_rows_count, |
1094 | 0 | TData1& res) { |
1095 | 0 | for (int i = 0; i < input_rows_count; ++i) { |
1096 | 0 | if (null_map[i]) { |
1097 | 0 | continue; |
1098 | 0 | } |
1099 | 0 | if (offset_data < 0 || limit_data < 0) { |
1100 | 0 | null_map[i] = 1; |
1101 | 0 | continue; |
1102 | 0 | } |
1103 | 0 | bitmap_data[i].sub_limit(offset_data, limit_data, &res[i]); |
1104 | 0 | } |
1105 | 0 | } |
1106 | | }; |
1107 | | |
1108 | | struct BitmapSubsetInRange { |
1109 | | static constexpr auto name = "bitmap_subset_in_range"; |
1110 | | using TData1 = std::vector<BitmapValue>; |
1111 | | using TData2 = typename ColumnInt64::Container; |
1112 | | |
1113 | | static void vector3(const TData1& bitmap_data, const TData2& range_start, |
1114 | | const TData2& range_end, NullMap& null_map, size_t input_rows_count, |
1115 | 11 | TData1& res) { |
1116 | 39 | for (int i = 0; i < input_rows_count; ++i) { |
1117 | 28 | if (null_map[i]) { |
1118 | 0 | continue; |
1119 | 0 | } |
1120 | 28 | if (range_start[i] >= range_end[i] || range_start[i] < 0 || range_end[i] < 0) { |
1121 | 25 | null_map[i] = 1; |
1122 | 25 | continue; |
1123 | 25 | } |
1124 | 3 | bitmap_data[i].sub_range(range_start[i], range_end[i], &res[i]); |
1125 | 3 | } |
1126 | 11 | } |
1127 | | static void vector_scalars(const TData1& bitmap_data, const Int64& range_start, |
1128 | | const Int64& range_end, NullMap& null_map, size_t input_rows_count, |
1129 | 0 | TData1& res) { |
1130 | 0 | for (int i = 0; i < input_rows_count; ++i) { |
1131 | 0 | if (null_map[i]) { |
1132 | 0 | continue; |
1133 | 0 | } |
1134 | 0 | if (range_start >= range_end || range_start < 0 || range_end < 0) { |
1135 | 0 | null_map[i] = 1; |
1136 | 0 | continue; |
1137 | 0 | } |
1138 | 0 | bitmap_data[i].sub_range(range_start, range_end, &res[i]); |
1139 | 0 | } |
1140 | 0 | } |
1141 | | }; |
1142 | | |
1143 | | template <typename Impl> |
1144 | | class FunctionBitmapSubs : public IFunction { |
1145 | | public: |
1146 | | static constexpr auto name = Impl::name; |
1147 | 3 | String get_name() const override { return name; }_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE8get_nameB5cxx11Ev Line | Count | Source | 1147 | 1 | String get_name() const override { return name; } |
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE8get_nameB5cxx11Ev Line | Count | Source | 1147 | 1 | String get_name() const override { return name; } |
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE8get_nameB5cxx11Ev Line | Count | Source | 1147 | 1 | String get_name() const override { return name; } |
|
1148 | | |
1149 | 44 | static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); }_ZN5doris18FunctionBitmapSubsINS_9SubBitmapEE6createEv Line | Count | Source | 1149 | 16 | static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); } |
_ZN5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE6createEv Line | Count | Source | 1149 | 14 | static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); } |
_ZN5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE6createEv Line | Count | Source | 1149 | 14 | static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); } |
|
1150 | | |
1151 | 17 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { |
1152 | 17 | return make_nullable(std::make_shared<DataTypeBitMap>()); |
1153 | 17 | } _ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 1151 | 7 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 1152 | 7 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 1153 | 7 | } |
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 1151 | 5 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 1152 | 5 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 1153 | 5 | } |
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE Line | Count | Source | 1151 | 5 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { | 1152 | 5 | return make_nullable(std::make_shared<DataTypeBitMap>()); | 1153 | 5 | } |
|
1154 | | |
1155 | 17 | size_t get_number_of_arguments() const override { return 3; }_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE23get_number_of_argumentsEv Line | Count | Source | 1155 | 7 | size_t get_number_of_arguments() const override { return 3; } |
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE23get_number_of_argumentsEv Line | Count | Source | 1155 | 5 | size_t get_number_of_arguments() const override { return 3; } |
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE23get_number_of_argumentsEv Line | Count | Source | 1155 | 5 | size_t get_number_of_arguments() const override { return 3; } |
|
1156 | | |
1157 | | Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
1158 | 34 | uint32_t result, size_t input_rows_count) const override { |
1159 | 34 | DCHECK_EQ(arguments.size(), 3); |
1160 | 34 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); |
1161 | 34 | auto res_data_column = ColumnBitmap::create(input_rows_count); |
1162 | | |
1163 | 34 | bool col_const[3]; |
1164 | 34 | ColumnPtr argument_columns[3]; |
1165 | 139 | for (int i = 0; i < 3; ++i) { |
1166 | 105 | col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column); |
1167 | 105 | } |
1168 | 34 | argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>( |
1169 | 0 | *block.get_by_position(arguments[0]).column) |
1170 | 0 | .convert_to_full_column() |
1171 | 34 | : block.get_by_position(arguments[0]).column; |
1172 | | |
1173 | 34 | default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments); |
1174 | | |
1175 | 34 | auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get()); |
1176 | 34 | auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get()); |
1177 | 34 | auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get()); |
1178 | | |
1179 | 34 | if (col_const[1] && col_const[2]) { |
1180 | 0 | Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0), |
1181 | 0 | limit_column->get_element(0), res_null_map->get_data(), |
1182 | 0 | input_rows_count, res_data_column->get_data()); |
1183 | 34 | } else { |
1184 | 34 | Impl::vector3(bitmap_column->get_data(), offset_column->get_data(), |
1185 | 34 | limit_column->get_data(), res_null_map->get_data(), input_rows_count, |
1186 | 34 | res_data_column->get_data()); |
1187 | 34 | } |
1188 | | |
1189 | 34 | block.get_by_position(result).column = |
1190 | 34 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); |
1191 | 34 | return Status::OK(); |
1192 | 34 | } _ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 1158 | 13 | uint32_t result, size_t input_rows_count) const override { | 1159 | 13 | DCHECK_EQ(arguments.size(), 3); | 1160 | 13 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 1161 | 13 | auto res_data_column = ColumnBitmap::create(input_rows_count); | 1162 | | | 1163 | 13 | bool col_const[3]; | 1164 | 13 | ColumnPtr argument_columns[3]; | 1165 | 52 | for (int i = 0; i < 3; ++i) { | 1166 | 39 | col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column); | 1167 | 39 | } | 1168 | 13 | argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>( | 1169 | 0 | *block.get_by_position(arguments[0]).column) | 1170 | 0 | .convert_to_full_column() | 1171 | 13 | : block.get_by_position(arguments[0]).column; | 1172 | | | 1173 | 13 | default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments); | 1174 | | | 1175 | 13 | auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get()); | 1176 | 13 | auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get()); | 1177 | 13 | auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get()); | 1178 | | | 1179 | 13 | if (col_const[1] && col_const[2]) { | 1180 | 0 | Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0), | 1181 | 0 | limit_column->get_element(0), res_null_map->get_data(), | 1182 | 0 | input_rows_count, res_data_column->get_data()); | 1183 | 13 | } else { | 1184 | 13 | Impl::vector3(bitmap_column->get_data(), offset_column->get_data(), | 1185 | 13 | limit_column->get_data(), res_null_map->get_data(), input_rows_count, | 1186 | 13 | res_data_column->get_data()); | 1187 | 13 | } | 1188 | | | 1189 | 13 | block.get_by_position(result).column = | 1190 | 13 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 1191 | 13 | return Status::OK(); | 1192 | 13 | } |
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 1158 | 11 | uint32_t result, size_t input_rows_count) const override { | 1159 | 11 | DCHECK_EQ(arguments.size(), 3); | 1160 | 11 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 1161 | 11 | auto res_data_column = ColumnBitmap::create(input_rows_count); | 1162 | | | 1163 | 11 | bool col_const[3]; | 1164 | 11 | ColumnPtr argument_columns[3]; | 1165 | 44 | for (int i = 0; i < 3; ++i) { | 1166 | 33 | col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column); | 1167 | 33 | } | 1168 | 11 | argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>( | 1169 | 0 | *block.get_by_position(arguments[0]).column) | 1170 | 0 | .convert_to_full_column() | 1171 | 11 | : block.get_by_position(arguments[0]).column; | 1172 | | | 1173 | 11 | default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments); | 1174 | | | 1175 | 11 | auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get()); | 1176 | 11 | auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get()); | 1177 | 11 | auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get()); | 1178 | | | 1179 | 11 | if (col_const[1] && col_const[2]) { | 1180 | 0 | Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0), | 1181 | 0 | limit_column->get_element(0), res_null_map->get_data(), | 1182 | 0 | input_rows_count, res_data_column->get_data()); | 1183 | 11 | } else { | 1184 | 11 | Impl::vector3(bitmap_column->get_data(), offset_column->get_data(), | 1185 | 11 | limit_column->get_data(), res_null_map->get_data(), input_rows_count, | 1186 | 11 | res_data_column->get_data()); | 1187 | 11 | } | 1188 | | | 1189 | 11 | block.get_by_position(result).column = | 1190 | 11 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 1191 | 11 | return Status::OK(); | 1192 | 11 | } |
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm Line | Count | Source | 1158 | 10 | uint32_t result, size_t input_rows_count) const override { | 1159 | 10 | DCHECK_EQ(arguments.size(), 3); | 1160 | 10 | auto res_null_map = ColumnUInt8::create(input_rows_count, 0); | 1161 | 10 | auto res_data_column = ColumnBitmap::create(input_rows_count); | 1162 | | | 1163 | 10 | bool col_const[3]; | 1164 | 10 | ColumnPtr argument_columns[3]; | 1165 | 43 | for (int i = 0; i < 3; ++i) { | 1166 | 33 | col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column); | 1167 | 33 | } | 1168 | 10 | argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>( | 1169 | 0 | *block.get_by_position(arguments[0]).column) | 1170 | 0 | .convert_to_full_column() | 1171 | 10 | : block.get_by_position(arguments[0]).column; | 1172 | | | 1173 | 10 | default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments); | 1174 | | | 1175 | 10 | auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get()); | 1176 | 10 | auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get()); | 1177 | 10 | auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get()); | 1178 | | | 1179 | 10 | if (col_const[1] && col_const[2]) { | 1180 | 0 | Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0), | 1181 | 0 | limit_column->get_element(0), res_null_map->get_data(), | 1182 | 0 | input_rows_count, res_data_column->get_data()); | 1183 | 10 | } else { | 1184 | 10 | Impl::vector3(bitmap_column->get_data(), offset_column->get_data(), | 1185 | 10 | limit_column->get_data(), res_null_map->get_data(), input_rows_count, | 1186 | 10 | res_data_column->get_data()); | 1187 | 10 | } | 1188 | | | 1189 | 10 | block.get_by_position(result).column = | 1190 | 10 | ColumnNullable::create(std::move(res_data_column), std::move(res_null_map)); | 1191 | 10 | return Status::OK(); | 1192 | 10 | } |
|
1193 | | }; |
1194 | | |
1195 | | class FunctionBitmapToArray : public IFunction { |
1196 | | public: |
1197 | | static constexpr auto name = "bitmap_to_array"; |
1198 | | |
1199 | 1 | String get_name() const override { return name; } |
1200 | | |
1201 | 14 | static FunctionPtr create() { return std::make_shared<FunctionBitmapToArray>(); } |
1202 | | |
1203 | 5 | DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { |
1204 | 5 | auto nested_type = make_nullable(std::make_shared<DataTypeInt64>()); |
1205 | 5 | return std::make_shared<DataTypeArray>(nested_type); |
1206 | 5 | } |
1207 | | |
1208 | 5 | size_t get_number_of_arguments() const override { return 1; } |
1209 | | |
1210 | | Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, |
1211 | 8 | uint32_t result, size_t input_rows_count) const override { |
1212 | 8 | auto return_nested_type = make_nullable(std::make_shared<DataTypeInt64>()); |
1213 | 8 | auto dest_array_column_ptr = ColumnArray::create(return_nested_type->create_column(), |
1214 | 8 | ColumnArray::ColumnOffsets::create()); |
1215 | | |
1216 | 8 | IColumn* dest_nested_column = &dest_array_column_ptr->get_data(); |
1217 | 8 | ColumnNullable* dest_nested_nullable_col = |
1218 | 8 | reinterpret_cast<ColumnNullable*>(dest_nested_column); |
1219 | 8 | dest_nested_column = dest_nested_nullable_col->get_nested_column_ptr().get(); |
1220 | 8 | auto& dest_nested_null_map = dest_nested_nullable_col->get_null_map_column().get_data(); |
1221 | | |
1222 | 8 | auto& arg_col = block.get_by_position(arguments[0]).column; |
1223 | 8 | auto bitmap_col = assert_cast<const ColumnBitmap*>(arg_col.get()); |
1224 | 8 | const auto& bitmap_col_data = bitmap_col->get_data(); |
1225 | 8 | auto& nested_column_data = assert_cast<ColumnInt64*>(dest_nested_column)->get_data(); |
1226 | 8 | auto& dest_offsets = dest_array_column_ptr->get_offsets(); |
1227 | 8 | dest_offsets.reserve(input_rows_count); |
1228 | | |
1229 | 16 | for (int i = 0; i < input_rows_count; ++i) { |
1230 | 8 | bitmap_col_data[i].to_array(nested_column_data); |
1231 | 8 | dest_nested_null_map.resize_fill(nested_column_data.size(), 0); |
1232 | 8 | dest_offsets.push_back(nested_column_data.size()); |
1233 | 8 | } |
1234 | | |
1235 | 8 | block.replace_by_position(result, std::move(dest_array_column_ptr)); |
1236 | 8 | return Status::OK(); |
1237 | 8 | } |
1238 | | }; |
1239 | | |
1240 | | using FunctionBitmapEmpty = FunctionConst<BitmapEmpty, false>; |
1241 | | using FunctionToBitmap = FunctionAlwaysNotNullable<ToBitmap>; |
1242 | | using FunctionToBitmapWithCheck = FunctionAlwaysNotNullable<ToBitmapWithCheck, true>; |
1243 | | |
1244 | | using FunctionBitmapFromString = FunctionBitmapAlwaysNull<BitmapFromString>; |
1245 | | using FunctionBitmapFromArray = FunctionBitmapAlwaysNull<BitmapFromArray>; |
1246 | | using FunctionBitmapHash = FunctionAlwaysNotNullable<BitmapHash<32>>; |
1247 | | using FunctionBitmapHash64 = FunctionAlwaysNotNullable<BitmapHash<64>>; |
1248 | | |
1249 | | using FunctionBitmapMin = FunctionBitmapSingle<FunctionBitmapMinImpl>; |
1250 | | using FunctionBitmapMax = FunctionBitmapSingle<FunctionBitmapMaxImpl>; |
1251 | | |
1252 | | using FunctionBitmapToString = FunctionUnaryToType<BitmapToString, NameBitmapToString>; |
1253 | | using FunctionBitmapToBase64 = FunctionUnaryToType<BitmapToBase64, NameBitmapToBase64>; |
1254 | | using FunctionBitmapFromBase64 = FunctionBitmapAlwaysNull<BitmapFromBase64>; |
1255 | | using FunctionBitmapNot = |
1256 | | FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapNot, NameBitmapNot>; |
1257 | | using FunctionBitmapAndNot = |
1258 | | FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapAndNot, NameBitmapAndNot>; |
1259 | | using FunctionBitmapContains = |
1260 | | FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapContains, NameBitmapContains>; |
1261 | | using FunctionBitmapRemove = |
1262 | | FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapRemove, NameBitmapRemove>; |
1263 | | |
1264 | | using FunctionBitmapHasAny = |
1265 | | FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAny, NameBitmapHasAny>; |
1266 | | using FunctionBitmapHasAll = |
1267 | | FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAll, NameBitmapHasAll>; |
1268 | | using FunctionSubBitmap = FunctionBitmapSubs<SubBitmap>; |
1269 | | using FunctionBitmapSubsetLimit = FunctionBitmapSubs<BitmapSubsetLimit>; |
1270 | | using FunctionBitmapSubsetInRange = FunctionBitmapSubs<BitmapSubsetInRange>; |
1271 | | |
1272 | 8 | void register_function_bitmap(SimpleFunctionFactory& factory) { |
1273 | 8 | factory.register_function<FunctionBitmapEmpty>(); |
1274 | 8 | factory.register_function<FunctionToBitmap>(); |
1275 | 8 | factory.register_function<FunctionToBitmapWithCheck>(); |
1276 | 8 | factory.register_function<FunctionBitmapFromString>(); |
1277 | 8 | factory.register_function<FunctionBitmapToBase64>(); |
1278 | 8 | factory.register_function<FunctionBitmapFromBase64>(); |
1279 | 8 | factory.register_function<FunctionBitmapFromArray>(); |
1280 | 8 | factory.register_function<FunctionBitmapHash>(); |
1281 | 8 | factory.register_function<FunctionBitmapHash64>(); |
1282 | 8 | factory.register_function<FunctionBitmapCount>(); |
1283 | 8 | factory.register_function<FunctionBitmapMin>(); |
1284 | 8 | factory.register_function<FunctionBitmapMax>(); |
1285 | 8 | factory.register_function<FunctionBitmapToString>(); |
1286 | 8 | factory.register_function<FunctionBitmapNot>(); |
1287 | 8 | factory.register_function<FunctionBitmapAndNot>(); |
1288 | 8 | factory.register_alias(NameBitmapAndNot::name, "bitmap_andnot"); |
1289 | 8 | factory.register_function<FunctionBitmapAndNotCount<NameBitmapAndNotCount>>(); |
1290 | 8 | factory.register_alias(NameBitmapAndNotCount::name, "bitmap_andnot_count"); |
1291 | 8 | factory.register_function<FunctionBitmapContains>(); |
1292 | 8 | factory.register_function<FunctionBitmapRemove>(); |
1293 | 8 | factory.register_function<FunctionBitmapHasAny>(); |
1294 | 8 | factory.register_function<FunctionBitmapHasAll>(); |
1295 | 8 | factory.register_function<FunctionSubBitmap>(); |
1296 | 8 | factory.register_function<FunctionBitmapSubsetLimit>(); |
1297 | 8 | factory.register_function<FunctionBitmapSubsetInRange>(); |
1298 | 8 | factory.register_function<FunctionBitmapToArray>(); |
1299 | 8 | } |
1300 | | |
1301 | | } // namespace doris |