Coverage Report

Created: 2026-07-16 15:25

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exprs/function/function_bitmap.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
// This file is copied from
18
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionBitmap.h
19
// and modified by Doris
20
21
#include <absl/strings/numbers.h>
22
#include <absl/strings/str_split.h>
23
#include <glog/logging.h>
24
#include <stdint.h>
25
#include <string.h>
26
27
#include <algorithm>
28
#include <boost/iterator/iterator_facade.hpp>
29
#include <functional>
30
#include <memory>
31
#include <ostream>
32
#include <string>
33
#include <utility>
34
#include <vector>
35
36
#include "common/compiler_util.h" // IWYU pragma: keep
37
#include "common/status.h"
38
#include "core/assert_cast.h"
39
#include "core/block/block.h"
40
#include "core/block/column_numbers.h"
41
#include "core/block/column_with_type_and_name.h"
42
#include "core/column/column.h"
43
#include "core/column/column_array.h"
44
#include "core/column/column_complex.h"
45
#include "core/column/column_const.h"
46
#include "core/column/column_nullable.h"
47
#include "core/column/column_string.h"
48
#include "core/column/column_vector.h"
49
#include "core/data_type/data_type.h"
50
#include "core/data_type/data_type_array.h"
51
#include "core/data_type/data_type_bitmap.h"
52
#include "core/data_type/data_type_nullable.h"
53
#include "core/data_type/data_type_number.h"
54
#include "core/data_type/data_type_string.h"
55
#include "core/field.h"
56
#include "core/types.h"
57
#include "core/value/bitmap_value.h"
58
#include "exec/common/stringop_substring.h"
59
#include "exec/common/util.hpp"
60
#include "exprs/aggregate/aggregate_function.h"
61
#include "exprs/function/function.h"
62
#include "exprs/function/function_always_not_nullable.h"
63
#include "exprs/function/function_bitmap_min_or_max.h"
64
#include "exprs/function/function_const.h"
65
#include "exprs/function/function_helpers.h"
66
#include "exprs/function/function_totype.h"
67
#include "exprs/function/simple_function_factory.h"
68
#include "util/hash/murmur_hash3.h"
69
#include "util/hash_util.hpp"
70
#include "util/string_parser.hpp"
71
#include "util/url_coding.h"
72
73
namespace doris {
74
class FunctionContext;
75
} // namespace doris
76
77
namespace doris {
78
79
struct BitmapEmpty {
80
    static constexpr auto name = "bitmap_empty";
81
    using ReturnColVec = ColumnBitmap;
82
272
    static DataTypePtr get_return_type() { return std::make_shared<DataTypeBitMap>(); }
83
272
    static auto init_value() { return BitmapValue {}; }
84
};
85
86
struct ToBitmap {
87
    static constexpr auto name = "to_bitmap";
88
    using ReturnType = DataTypeBitMap;
89
90
    template <typename ColumnType>
91
1.08k
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
92
1.08k
        execute<ColumnType, false>(col, nullptr, col_res);
93
1.08k
    }
_ZN5doris8ToBitmap6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS8_EE
Line
Count
Source
91
12
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
92
12
        execute<ColumnType, false>(col, nullptr, col_res);
93
12
    }
_ZN5doris8ToBitmap6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE
Line
Count
Source
91
1.07k
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
92
1.07k
        execute<ColumnType, false>(col, nullptr, col_res);
93
1.07k
    }
94
    template <typename ColumnType>
95
    static void vector_nullable(const ColumnType* col, const NullMap& nullmap,
96
532
                                MutableColumnPtr& col_res) {
97
532
        execute<ColumnType, true>(col, &nullmap, col_res);
98
532
    }
_ZN5doris8ToBitmap15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE
Line
Count
Source
96
51
                                MutableColumnPtr& col_res) {
97
51
        execute<ColumnType, true>(col, &nullmap, col_res);
98
51
    }
_ZN5doris8ToBitmap15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
Line
Count
Source
96
481
                                MutableColumnPtr& col_res) {
97
481
        execute<ColumnType, true>(col, &nullmap, col_res);
98
481
    }
99
    template <typename ColumnType, bool arg_is_nullable>
100
1.61k
    static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) {
101
1.61k
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
102
63
            const ColumnString::Chars& data = col->get_chars();
103
63
            const ColumnString::Offsets& offsets = col->get_offsets();
104
105
63
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
106
63
            auto& res_data = res_column->get_data();
107
63
            size_t size = offsets.size();
108
109
3.57k
            for (size_t i = 0; i < size; ++i) {
110
3.50k
                if (arg_is_nullable && ((*nullmap)[i])) {
111
34
                    continue;
112
3.47k
                } else {
113
3.47k
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
114
3.47k
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
115
3.47k
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
116
3.47k
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
117
3.47k
                            raw_str, str_size, &parse_result);
118
3.47k
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
119
3.26k
                        res_data[i].add(int_value);
120
3.26k
                    }
121
3.47k
                }
122
3.50k
            }
123
1.55k
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
124
1.55k
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
125
1.55k
            auto& res_data = res_column->get_data();
126
1.55k
            size_t size = col->size();
127
128
141k
            for (size_t i = 0; i < size; ++i) {
129
139k
                if constexpr (arg_is_nullable) {
130
1.94k
                    if ((*nullmap)[i]) {
131
830
                        continue;
132
830
                    }
133
1.94k
                }
134
139k
                if (auto value = col->get_data()[i]; value >= 0) {
135
133k
                    res_data[i].add(value);
136
133k
                }
137
139k
            }
138
1.55k
        }
139
1.61k
    }
_ZN5doris8ToBitmap7executeINS_9ColumnStrIjEELb1EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE
Line
Count
Source
100
51
    static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) {
101
51
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
102
51
            const ColumnString::Chars& data = col->get_chars();
103
51
            const ColumnString::Offsets& offsets = col->get_offsets();
104
105
51
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
106
51
            auto& res_data = res_column->get_data();
107
51
            size_t size = offsets.size();
108
109
3.53k
            for (size_t i = 0; i < size; ++i) {
110
3.47k
                if (arg_is_nullable && ((*nullmap)[i])) {
111
34
                    continue;
112
3.44k
                } else {
113
3.44k
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
114
3.44k
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
115
3.44k
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
116
3.44k
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
117
3.44k
                            raw_str, str_size, &parse_result);
118
3.44k
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
119
3.26k
                        res_data[i].add(int_value);
120
3.26k
                    }
121
3.44k
                }
122
3.47k
            }
123
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
124
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
125
            auto& res_data = res_column->get_data();
126
            size_t size = col->size();
127
128
            for (size_t i = 0; i < size; ++i) {
129
                if constexpr (arg_is_nullable) {
130
                    if ((*nullmap)[i]) {
131
                        continue;
132
                    }
133
                }
134
                if (auto value = col->get_data()[i]; value >= 0) {
135
                    res_data[i].add(value);
136
                }
137
            }
138
        }
139
51
    }
_ZN5doris8ToBitmap7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb1EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
Line
Count
Source
100
481
    static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) {
101
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
102
            const ColumnString::Chars& data = col->get_chars();
103
            const ColumnString::Offsets& offsets = col->get_offsets();
104
105
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
106
            auto& res_data = res_column->get_data();
107
            size_t size = offsets.size();
108
109
            for (size_t i = 0; i < size; ++i) {
110
                if (arg_is_nullable && ((*nullmap)[i])) {
111
                    continue;
112
                } else {
113
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
114
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
115
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
116
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
117
                            raw_str, str_size, &parse_result);
118
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
119
                        res_data[i].add(int_value);
120
                    }
121
                }
122
            }
123
481
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
124
481
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
125
481
            auto& res_data = res_column->get_data();
126
481
            size_t size = col->size();
127
128
3.26k
            for (size_t i = 0; i < size; ++i) {
129
1.94k
                if constexpr (arg_is_nullable) {
130
1.94k
                    if ((*nullmap)[i]) {
131
830
                        continue;
132
830
                    }
133
1.94k
                }
134
1.94k
                if (auto value = col->get_data()[i]; value >= 0) {
135
785
                    res_data[i].add(value);
136
785
                }
137
1.94k
            }
138
481
        }
139
481
    }
_ZN5doris8ToBitmap7executeINS_9ColumnStrIjEELb0EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISF_EE
Line
Count
Source
100
12
    static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) {
101
12
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
102
12
            const ColumnString::Chars& data = col->get_chars();
103
12
            const ColumnString::Offsets& offsets = col->get_offsets();
104
105
12
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
106
12
            auto& res_data = res_column->get_data();
107
12
            size_t size = offsets.size();
108
109
40
            for (size_t i = 0; i < size; ++i) {
110
28
                if (arg_is_nullable && ((*nullmap)[i])) {
111
0
                    continue;
112
28
                } else {
113
28
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
114
28
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
115
28
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
116
28
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
117
28
                            raw_str, str_size, &parse_result);
118
28
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
119
0
                        res_data[i].add(int_value);
120
0
                    }
121
28
                }
122
28
            }
123
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
124
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
125
            auto& res_data = res_column->get_data();
126
            size_t size = col->size();
127
128
            for (size_t i = 0; i < size; ++i) {
129
                if constexpr (arg_is_nullable) {
130
                    if ((*nullmap)[i]) {
131
                        continue;
132
                    }
133
                }
134
                if (auto value = col->get_data()[i]; value >= 0) {
135
                    res_data[i].add(value);
136
                }
137
            }
138
        }
139
12
    }
_ZN5doris8ToBitmap7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb0EEEvPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
Line
Count
Source
100
1.07k
    static void execute(const ColumnType* col, const NullMap* nullmap, MutableColumnPtr& col_res) {
101
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
102
            const ColumnString::Chars& data = col->get_chars();
103
            const ColumnString::Offsets& offsets = col->get_offsets();
104
105
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
106
            auto& res_data = res_column->get_data();
107
            size_t size = offsets.size();
108
109
            for (size_t i = 0; i < size; ++i) {
110
                if (arg_is_nullable && ((*nullmap)[i])) {
111
                    continue;
112
                } else {
113
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
114
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
115
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
116
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
117
                            raw_str, str_size, &parse_result);
118
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
119
                        res_data[i].add(int_value);
120
                    }
121
                }
122
            }
123
1.07k
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
124
1.07k
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
125
1.07k
            auto& res_data = res_column->get_data();
126
1.07k
            size_t size = col->size();
127
128
138k
            for (size_t i = 0; i < size; ++i) {
129
                if constexpr (arg_is_nullable) {
130
                    if ((*nullmap)[i]) {
131
                        continue;
132
                    }
133
                }
134
137k
                if (auto value = col->get_data()[i]; value >= 0) {
135
133k
                    res_data[i].add(value);
136
133k
                }
137
137k
            }
138
1.07k
        }
139
1.07k
    }
140
};
141
142
struct ToBitmapWithCheck {
143
    static constexpr auto name = "to_bitmap_with_check";
144
    using ReturnType = DataTypeBitMap;
145
146
    template <typename ColumnType>
147
52
    static Status vector(const ColumnType* col, MutableColumnPtr& col_res) {
148
52
        return execute<ColumnType, false>(col, nullptr, col_res);
149
52
    }
Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck6vectorINS_9ColumnStrIjEEEENS_6StatusEPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE
_ZN5doris17ToBitmapWithCheck6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusEPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE
Line
Count
Source
147
52
    static Status vector(const ColumnType* col, MutableColumnPtr& col_res) {
148
52
        return execute<ColumnType, false>(col, nullptr, col_res);
149
52
    }
150
    template <typename ColumnType>
151
    static Status vector_nullable(const ColumnType* col, const NullMap& nullmap,
152
51
                                  MutableColumnPtr& col_res) {
153
51
        return execute<ColumnType, true>(col, &nullmap, col_res);
154
51
    }
Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck15vector_nullableINS_9ColumnStrIjEEEENS_6StatusEPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
_ZN5doris17ToBitmapWithCheck15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusEPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE
Line
Count
Source
152
51
                                  MutableColumnPtr& col_res) {
153
51
        return execute<ColumnType, true>(col, &nullmap, col_res);
154
51
    }
155
    template <typename ColumnType, bool arg_is_nullable>
156
    static Status execute(const ColumnType* col, const NullMap* nullmap,
157
103
                          MutableColumnPtr& col_res) {
158
103
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
159
0
            const ColumnString::Chars& data = col->get_chars();
160
0
            const ColumnString::Offsets& offsets = col->get_offsets();
161
0
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
162
0
            auto& res_data = res_column->get_data();
163
0
            size_t size = offsets.size();
164
165
0
            for (size_t i = 0; i < size; ++i) {
166
0
                if (arg_is_nullable && ((*nullmap)[i])) {
167
0
                    continue;
168
0
                } else {
169
0
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
170
                    // The string lenght is less than 2G, so that cast the str size to int, not use size_t
171
0
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
172
0
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
173
0
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
174
0
                            raw_str, str_size, &parse_result);
175
0
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
176
0
                        res_data[i].add(int_value);
177
0
                    } else {
178
0
                        return Status::InvalidArgument(
179
0
                                "The input: {} is not valid, to_bitmap only support bigint value "
180
0
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
181
0
                                "to_bitmap on column with negative values or cannot load negative "
182
0
                                "values to column with to_bitmap MV on it.",
183
0
                                std::string(raw_str, str_size));
184
0
                    }
185
0
                }
186
0
            }
187
103
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
188
103
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
189
103
            auto& res_data = res_column->get_data();
190
103
            size_t size = col->size();
191
192
230
            for (size_t i = 0; i < size; ++i) {
193
130
                if (arg_is_nullable && ((*nullmap)[i])) {
194
21
                    continue;
195
109
                } else {
196
109
                    int64_t int_value = col->get_data()[i];
197
109
                    if (LIKELY(int_value >= 0)) {
198
106
                        res_data[i].add(int_value);
199
106
                    } else {
200
3
                        return Status::InvalidArgument(
201
3
                                "The input: {} is not valid, to_bitmap only support bigint value "
202
3
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
203
3
                                "to_bitmap on column with negative values or cannot load negative "
204
3
                                "values to column with to_bitmap MV on it.",
205
3
                                int_value);
206
3
                    }
207
109
                }
208
130
            }
209
        } else {
210
            return Status::InvalidArgument("not support type");
211
        }
212
100
        return Status::OK();
213
103
    }
Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck7executeINS_9ColumnStrIjEELb1EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
_ZN5doris17ToBitmapWithCheck7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb1EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE
Line
Count
Source
157
51
                          MutableColumnPtr& col_res) {
158
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
159
            const ColumnString::Chars& data = col->get_chars();
160
            const ColumnString::Offsets& offsets = col->get_offsets();
161
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
162
            auto& res_data = res_column->get_data();
163
            size_t size = offsets.size();
164
165
            for (size_t i = 0; i < size; ++i) {
166
                if (arg_is_nullable && ((*nullmap)[i])) {
167
                    continue;
168
                } else {
169
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
170
                    // The string lenght is less than 2G, so that cast the str size to int, not use size_t
171
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
172
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
173
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
174
                            raw_str, str_size, &parse_result);
175
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
176
                        res_data[i].add(int_value);
177
                    } else {
178
                        return Status::InvalidArgument(
179
                                "The input: {} is not valid, to_bitmap only support bigint value "
180
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
181
                                "to_bitmap on column with negative values or cannot load negative "
182
                                "values to column with to_bitmap MV on it.",
183
                                std::string(raw_str, str_size));
184
                    }
185
                }
186
            }
187
51
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
188
51
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
189
51
            auto& res_data = res_column->get_data();
190
51
            size_t size = col->size();
191
192
121
            for (size_t i = 0; i < size; ++i) {
193
70
                if (arg_is_nullable && ((*nullmap)[i])) {
194
21
                    continue;
195
49
                } else {
196
49
                    int64_t int_value = col->get_data()[i];
197
49
                    if (LIKELY(int_value >= 0)) {
198
49
                        res_data[i].add(int_value);
199
49
                    } else {
200
0
                        return Status::InvalidArgument(
201
0
                                "The input: {} is not valid, to_bitmap only support bigint value "
202
0
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
203
0
                                "to_bitmap on column with negative values or cannot load negative "
204
0
                                "values to column with to_bitmap MV on it.",
205
0
                                int_value);
206
0
                    }
207
49
                }
208
70
            }
209
        } else {
210
            return Status::InvalidArgument("not support type");
211
        }
212
51
        return Status::OK();
213
51
    }
Unexecuted instantiation: _ZN5doris17ToBitmapWithCheck7executeINS_9ColumnStrIjEELb0EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
_ZN5doris17ToBitmapWithCheck7executeINS_12ColumnVectorILNS_13PrimitiveTypeE6EEELb0EEENS_6StatusEPKT_PKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE
Line
Count
Source
157
52
                          MutableColumnPtr& col_res) {
158
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
159
            const ColumnString::Chars& data = col->get_chars();
160
            const ColumnString::Offsets& offsets = col->get_offsets();
161
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
162
            auto& res_data = res_column->get_data();
163
            size_t size = offsets.size();
164
165
            for (size_t i = 0; i < size; ++i) {
166
                if (arg_is_nullable && ((*nullmap)[i])) {
167
                    continue;
168
                } else {
169
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
170
                    // The string lenght is less than 2G, so that cast the str size to int, not use size_t
171
                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
172
                    StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
173
                    uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
174
                            raw_str, str_size, &parse_result);
175
                    if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) {
176
                        res_data[i].add(int_value);
177
                    } else {
178
                        return Status::InvalidArgument(
179
                                "The input: {} is not valid, to_bitmap only support bigint value "
180
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
181
                                "to_bitmap on column with negative values or cannot load negative "
182
                                "values to column with to_bitmap MV on it.",
183
                                std::string(raw_str, str_size));
184
                    }
185
                }
186
            }
187
52
        } else if constexpr (std::is_same_v<ColumnType, ColumnInt64>) {
188
52
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
189
52
            auto& res_data = res_column->get_data();
190
52
            size_t size = col->size();
191
192
109
            for (size_t i = 0; i < size; ++i) {
193
60
                if (arg_is_nullable && ((*nullmap)[i])) {
194
0
                    continue;
195
60
                } else {
196
60
                    int64_t int_value = col->get_data()[i];
197
60
                    if (LIKELY(int_value >= 0)) {
198
57
                        res_data[i].add(int_value);
199
57
                    } else {
200
3
                        return Status::InvalidArgument(
201
3
                                "The input: {} is not valid, to_bitmap only support bigint value "
202
3
                                "from 0 to 18446744073709551615 currently, cannot create MV with "
203
3
                                "to_bitmap on column with negative values or cannot load negative "
204
3
                                "values to column with to_bitmap MV on it.",
205
3
                                int_value);
206
3
                    }
207
60
                }
208
60
            }
209
        } else {
210
            return Status::InvalidArgument("not support type");
211
        }
212
49
        return Status::OK();
213
52
    }
214
};
215
216
struct BitmapFromString {
217
    using ArgumentType = DataTypeString;
218
219
    static constexpr auto name = "bitmap_from_string";
220
221
    static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets,
222
                         std::vector<BitmapValue>& res, NullMap& null_map,
223
281
                         size_t input_rows_count) {
224
281
        res.reserve(input_rows_count);
225
281
        std::vector<uint64_t> bits;
226
281
        if (offsets.size() == 0 && input_rows_count == 1) {
227
            // For NULL constant
228
0
            res.emplace_back();
229
0
            null_map[0] = 1;
230
0
            return Status::OK();
231
0
        }
232
233
370
        auto split_and_parse = [&bits](const char* raw_str, size_t str_size) {
234
370
            bits.clear();
235
370
            auto res = absl::StrSplit(std::string_view {raw_str, str_size}, ",", absl::SkipEmpty());
236
370
            uint64_t value = 0;
237
1.28k
            for (auto s : res) {
238
1.28k
                if (!absl::SimpleAtoi(s, &value)) {
239
66
                    return false;
240
66
                }
241
1.22k
                bits.push_back(value);
242
1.22k
            }
243
304
            return true;
244
370
        };
245
246
        // split by comma
247
248
651
        for (size_t i = 0; i < input_rows_count; ++i) {
249
370
            const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
250
370
            int64_t str_size = offsets[i] - offsets[i - 1];
251
252
370
            if ((str_size > INT32_MAX) || !split_and_parse(raw_str, str_size)) {
253
66
                res.emplace_back();
254
66
                null_map[i] = 1;
255
66
                continue;
256
66
            }
257
304
            res.emplace_back(bits);
258
304
        }
259
281
        return Status::OK();
260
281
    }
261
};
262
263
struct NameBitmapFromBase64 {
264
    static constexpr auto name = "bitmap_from_base64";
265
};
266
struct BitmapFromBase64 {
267
    using ArgumentType = DataTypeString;
268
269
    static constexpr auto name = "bitmap_from_base64";
270
271
    static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets,
272
                         std::vector<BitmapValue>& res, NullMap& null_map,
273
25
                         size_t input_rows_count) {
274
25
        res.reserve(input_rows_count);
275
25
        if (offsets.size() == 0 && input_rows_count == 1) {
276
            // For NULL constant
277
0
            res.emplace_back();
278
0
            null_map[0] = 1;
279
0
            return Status::OK();
280
0
        }
281
25
        std::string decode_buff;
282
25
        size_t last_decode_buff_len = 0;
283
25
        size_t curr_decode_buff_len = 0;
284
73
        for (size_t i = 0; i < input_rows_count; ++i) {
285
49
            const char* src_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
286
49
            size_t src_size = offsets[i] - offsets[i - 1];
287
49
            if (0 != src_size % 4) {
288
                // return Status::InvalidArgument(
289
                //         fmt::format("invalid base64: {}", std::string(src_str, src_size)));
290
0
                res.emplace_back();
291
0
                null_map[i] = 1;
292
0
                continue;
293
0
            }
294
49
            curr_decode_buff_len = src_size + 3;
295
49
            if (curr_decode_buff_len > last_decode_buff_len) {
296
37
                decode_buff.resize(curr_decode_buff_len);
297
37
                last_decode_buff_len = curr_decode_buff_len;
298
37
            }
299
49
            auto outlen = base64_decode(src_str, src_size, decode_buff.data());
300
49
            if (outlen < 0) {
301
0
                res.emplace_back();
302
0
                null_map[i] = 1;
303
49
            } else {
304
49
                BitmapValue bitmap_val;
305
49
                if (!bitmap_val.deserialize(decode_buff.data())) {
306
1
                    return Status::RuntimeError("bitmap_from_base64 decode failed: base64: {}",
307
1
                                                std::string(src_str, src_size));
308
1
                }
309
48
                res.emplace_back(std::move(bitmap_val));
310
48
            }
311
49
        }
312
24
        return Status::OK();
313
25
    }
314
};
315
struct BitmapFromArray {
316
    using ArgumentType = DataTypeArray;
317
    static constexpr auto name = "bitmap_from_array";
318
319
    template <typename ColumnType>
320
    static Status vector(const ColumnArray::Offsets64& offset_column_data,
321
                         const IColumn& nested_column, const NullMap& nested_null_map,
322
13
                         std::vector<BitmapValue>& res, NullMap& null_map) {
323
13
        const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data();
324
13
        auto size = offset_column_data.size();
325
13
        res.reserve(size);
326
        // Preserve the nested column's native integer type here.
327
        // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path
328
        // instead of widening every element to uint64_t first.
329
13
        using ValueType = typename ColumnType::value_type;
330
13
        std::vector<ValueType> bits;
331
36
        for (size_t i = 0; i < size; ++i) {
332
23
            auto curr_offset = offset_column_data[i];
333
23
            auto prev_offset = offset_column_data[i - 1];
334
23
            bits.reserve(curr_offset - prev_offset);
335
92
            for (auto j = prev_offset; j < curr_offset; ++j) {
336
71
                auto data = nested_column_data[j];
337
                // invaild value
338
71
                if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) {
339
2
                    res.emplace_back();
340
2
                    null_map[i] = 1;
341
2
                    break;
342
69
                } else {
343
69
                    bits.push_back(data);
344
69
                }
345
71
            }
346
            //input is valid value
347
23
            if (!null_map[i]) {
348
21
                res.emplace_back(bits);
349
21
            }
350
23
            bits.clear();
351
23
        }
352
13
        return Status::OK();
353
13
    }
_ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE3EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_
Line
Count
Source
322
3
                         std::vector<BitmapValue>& res, NullMap& null_map) {
323
3
        const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data();
324
3
        auto size = offset_column_data.size();
325
3
        res.reserve(size);
326
        // Preserve the nested column's native integer type here.
327
        // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path
328
        // instead of widening every element to uint64_t first.
329
3
        using ValueType = typename ColumnType::value_type;
330
3
        std::vector<ValueType> bits;
331
6
        for (size_t i = 0; i < size; ++i) {
332
3
            auto curr_offset = offset_column_data[i];
333
3
            auto prev_offset = offset_column_data[i - 1];
334
3
            bits.reserve(curr_offset - prev_offset);
335
16
            for (auto j = prev_offset; j < curr_offset; ++j) {
336
13
                auto data = nested_column_data[j];
337
                // invaild value
338
13
                if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) {
339
0
                    res.emplace_back();
340
0
                    null_map[i] = 1;
341
0
                    break;
342
13
                } else {
343
13
                    bits.push_back(data);
344
13
                }
345
13
            }
346
            //input is valid value
347
3
            if (!null_map[i]) {
348
3
                res.emplace_back(bits);
349
3
            }
350
3
            bits.clear();
351
3
        }
352
3
        return Status::OK();
353
3
    }
Unexecuted instantiation: _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE2EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_
Unexecuted instantiation: _ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE4EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_
_ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE5EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_
Line
Count
Source
322
3
                         std::vector<BitmapValue>& res, NullMap& null_map) {
323
3
        const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data();
324
3
        auto size = offset_column_data.size();
325
3
        res.reserve(size);
326
        // Preserve the nested column's native integer type here.
327
        // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path
328
        // instead of widening every element to uint64_t first.
329
3
        using ValueType = typename ColumnType::value_type;
330
3
        std::vector<ValueType> bits;
331
14
        for (size_t i = 0; i < size; ++i) {
332
11
            auto curr_offset = offset_column_data[i];
333
11
            auto prev_offset = offset_column_data[i - 1];
334
11
            bits.reserve(curr_offset - prev_offset);
335
56
            for (auto j = prev_offset; j < curr_offset; ++j) {
336
47
                auto data = nested_column_data[j];
337
                // invaild value
338
47
                if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) {
339
2
                    res.emplace_back();
340
2
                    null_map[i] = 1;
341
2
                    break;
342
45
                } else {
343
45
                    bits.push_back(data);
344
45
                }
345
47
            }
346
            //input is valid value
347
11
            if (!null_map[i]) {
348
9
                res.emplace_back(bits);
349
9
            }
350
11
            bits.clear();
351
11
        }
352
3
        return Status::OK();
353
3
    }
_ZN5doris15BitmapFromArray6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEENS_6StatusERKNS_8PODArrayImLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERKNS_7IColumnERKNS6_IhLm4096ES9_Lm16ELm15EEERSt6vectorINS_11BitmapValueESaISK_EERSG_
Line
Count
Source
322
7
                         std::vector<BitmapValue>& res, NullMap& null_map) {
323
7
        const auto& nested_column_data = static_cast<const ColumnType&>(nested_column).get_data();
324
7
        auto size = offset_column_data.size();
325
7
        res.reserve(size);
326
        // Preserve the nested column's native integer type here.
327
        // For Array<Int32>/Array<UInt32>-like inputs can reach the 32-bit `add_many` fast path
328
        // instead of widening every element to uint64_t first.
329
7
        using ValueType = typename ColumnType::value_type;
330
7
        std::vector<ValueType> bits;
331
16
        for (size_t i = 0; i < size; ++i) {
332
9
            auto curr_offset = offset_column_data[i];
333
9
            auto prev_offset = offset_column_data[i - 1];
334
9
            bits.reserve(curr_offset - prev_offset);
335
20
            for (auto j = prev_offset; j < curr_offset; ++j) {
336
11
                auto data = nested_column_data[j];
337
                // invaild value
338
11
                if (UNLIKELY(data < 0) || UNLIKELY(nested_null_map[j])) {
339
0
                    res.emplace_back();
340
0
                    null_map[i] = 1;
341
0
                    break;
342
11
                } else {
343
11
                    bits.push_back(data);
344
11
                }
345
11
            }
346
            //input is valid value
347
9
            if (!null_map[i]) {
348
9
                res.emplace_back(bits);
349
9
            }
350
9
            bits.clear();
351
9
        }
352
7
        return Status::OK();
353
7
    }
354
};
355
356
template <typename Impl>
357
class FunctionBitmapAlwaysNull : public IFunction {
358
public:
359
    static constexpr auto name = Impl::name;
360
361
3
    String get_name() const override { return name; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE8get_nameB5cxx11Ev
Line
Count
Source
361
1
    String get_name() const override { return name; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE8get_nameB5cxx11Ev
Line
Count
Source
361
1
    String get_name() const override { return name; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE8get_nameB5cxx11Ev
Line
Count
Source
361
1
    String get_name() const override { return name; }
362
363
332
    static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); }
_ZN5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE6createEv
Line
Count
Source
363
276
    static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); }
_ZN5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE6createEv
Line
Count
Source
363
33
    static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); }
_ZN5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE6createEv
Line
Count
Source
363
23
    static FunctionPtr create() { return std::make_shared<FunctionBitmapAlwaysNull>(); }
364
365
305
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
366
305
        return make_nullable(std::make_shared<DataTypeBitMap>());
367
305
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
365
267
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
366
267
        return make_nullable(std::make_shared<DataTypeBitMap>());
367
267
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
365
24
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
366
24
        return make_nullable(std::make_shared<DataTypeBitMap>());
367
24
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
365
14
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
366
14
        return make_nullable(std::make_shared<DataTypeBitMap>());
367
14
    }
368
369
305
    size_t get_number_of_arguments() const override { return 1; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE23get_number_of_argumentsEv
Line
Count
Source
369
267
    size_t get_number_of_arguments() const override { return 1; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE23get_number_of_argumentsEv
Line
Count
Source
369
24
    size_t get_number_of_arguments() const override { return 1; }
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE23get_number_of_argumentsEv
Line
Count
Source
369
14
    size_t get_number_of_arguments() const override { return 1; }
370
371
    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
372
319
                        uint32_t result, size_t input_rows_count) const override {
373
319
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
374
319
        auto res_data_column = ColumnBitmap::create();
375
319
        auto& null_map = res_null_map->get_data();
376
319
        auto& res = res_data_column->get_data();
377
378
319
        ColumnPtr& argument_column = block.get_by_position(arguments[0]).column;
379
319
        if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) {
380
306
            const auto& str_column = static_cast<const ColumnString&>(*argument_column);
381
306
            const ColumnString::Chars& data = str_column.get_chars();
382
306
            const ColumnString::Offsets& offsets = str_column.get_offsets();
383
306
            RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count));
384
306
        } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) {
385
13
            auto argument_type = remove_nullable(
386
13
                    assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type)
387
13
                            .get_nested_type());
388
13
            const auto& array_column = static_cast<const ColumnArray&>(*argument_column);
389
13
            const auto& offset_column_data = array_column.get_offsets();
390
13
            const auto& nested_nullable_column =
391
13
                    static_cast<const ColumnNullable&>(array_column.get_data());
392
13
            const auto& nested_column = nested_nullable_column.get_nested_column();
393
13
            const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data();
394
395
13
            switch (argument_type->get_primitive_type()) {
396
3
            case PrimitiveType::TYPE_TINYINT:
397
3
                RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column,
398
3
                                                                  nested_null_map, res, null_map));
399
3
                break;
400
3
            case PrimitiveType::TYPE_BOOLEAN:
401
0
                RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>(
402
0
                        offset_column_data, nested_column, nested_null_map, res, null_map));
403
0
                break;
404
0
            case PrimitiveType::TYPE_SMALLINT:
405
0
                RETURN_IF_ERROR(Impl::template vector<ColumnInt16>(
406
0
                        offset_column_data, nested_column, nested_null_map, res, null_map));
407
0
                break;
408
3
            case PrimitiveType::TYPE_INT:
409
3
                RETURN_IF_ERROR(Impl::template vector<ColumnInt32>(
410
3
                        offset_column_data, nested_column, nested_null_map, res, null_map));
411
3
                break;
412
7
            case PrimitiveType::TYPE_BIGINT:
413
7
                RETURN_IF_ERROR(Impl::template vector<ColumnInt64>(
414
7
                        offset_column_data, nested_column, nested_null_map, res, null_map));
415
7
                break;
416
7
            default:
417
0
                return Status::RuntimeError("Illegal column {} of argument of function {}",
418
0
                                            block.get_by_position(arguments[0]).column->get_name(),
419
0
                                            get_name());
420
13
            }
421
        } else {
422
            return Status::RuntimeError("Illegal column {} of argument of function {}",
423
                                        block.get_by_position(arguments[0]).column->get_name(),
424
                                        get_name());
425
        }
426
318
        block.get_by_position(result).column =
427
319
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
428
319
        return Status::OK();
429
319
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromStringEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
372
281
                        uint32_t result, size_t input_rows_count) const override {
373
281
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
374
281
        auto res_data_column = ColumnBitmap::create();
375
281
        auto& null_map = res_null_map->get_data();
376
281
        auto& res = res_data_column->get_data();
377
378
281
        ColumnPtr& argument_column = block.get_by_position(arguments[0]).column;
379
281
        if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) {
380
281
            const auto& str_column = static_cast<const ColumnString&>(*argument_column);
381
281
            const ColumnString::Chars& data = str_column.get_chars();
382
281
            const ColumnString::Offsets& offsets = str_column.get_offsets();
383
281
            RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count));
384
        } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) {
385
            auto argument_type = remove_nullable(
386
                    assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type)
387
                            .get_nested_type());
388
            const auto& array_column = static_cast<const ColumnArray&>(*argument_column);
389
            const auto& offset_column_data = array_column.get_offsets();
390
            const auto& nested_nullable_column =
391
                    static_cast<const ColumnNullable&>(array_column.get_data());
392
            const auto& nested_column = nested_nullable_column.get_nested_column();
393
            const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data();
394
395
            switch (argument_type->get_primitive_type()) {
396
            case PrimitiveType::TYPE_TINYINT:
397
                RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column,
398
                                                                  nested_null_map, res, null_map));
399
                break;
400
            case PrimitiveType::TYPE_BOOLEAN:
401
                RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>(
402
                        offset_column_data, nested_column, nested_null_map, res, null_map));
403
                break;
404
            case PrimitiveType::TYPE_SMALLINT:
405
                RETURN_IF_ERROR(Impl::template vector<ColumnInt16>(
406
                        offset_column_data, nested_column, nested_null_map, res, null_map));
407
                break;
408
            case PrimitiveType::TYPE_INT:
409
                RETURN_IF_ERROR(Impl::template vector<ColumnInt32>(
410
                        offset_column_data, nested_column, nested_null_map, res, null_map));
411
                break;
412
            case PrimitiveType::TYPE_BIGINT:
413
                RETURN_IF_ERROR(Impl::template vector<ColumnInt64>(
414
                        offset_column_data, nested_column, nested_null_map, res, null_map));
415
                break;
416
            default:
417
                return Status::RuntimeError("Illegal column {} of argument of function {}",
418
                                            block.get_by_position(arguments[0]).column->get_name(),
419
                                            get_name());
420
            }
421
        } else {
422
            return Status::RuntimeError("Illegal column {} of argument of function {}",
423
                                        block.get_by_position(arguments[0]).column->get_name(),
424
                                        get_name());
425
        }
426
281
        block.get_by_position(result).column =
427
281
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
428
281
        return Status::OK();
429
281
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_16BitmapFromBase64EE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
372
25
                        uint32_t result, size_t input_rows_count) const override {
373
25
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
374
25
        auto res_data_column = ColumnBitmap::create();
375
25
        auto& null_map = res_null_map->get_data();
376
25
        auto& res = res_data_column->get_data();
377
378
25
        ColumnPtr& argument_column = block.get_by_position(arguments[0]).column;
379
25
        if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) {
380
25
            const auto& str_column = static_cast<const ColumnString&>(*argument_column);
381
25
            const ColumnString::Chars& data = str_column.get_chars();
382
25
            const ColumnString::Offsets& offsets = str_column.get_offsets();
383
25
            RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count));
384
        } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) {
385
            auto argument_type = remove_nullable(
386
                    assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type)
387
                            .get_nested_type());
388
            const auto& array_column = static_cast<const ColumnArray&>(*argument_column);
389
            const auto& offset_column_data = array_column.get_offsets();
390
            const auto& nested_nullable_column =
391
                    static_cast<const ColumnNullable&>(array_column.get_data());
392
            const auto& nested_column = nested_nullable_column.get_nested_column();
393
            const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data();
394
395
            switch (argument_type->get_primitive_type()) {
396
            case PrimitiveType::TYPE_TINYINT:
397
                RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column,
398
                                                                  nested_null_map, res, null_map));
399
                break;
400
            case PrimitiveType::TYPE_BOOLEAN:
401
                RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>(
402
                        offset_column_data, nested_column, nested_null_map, res, null_map));
403
                break;
404
            case PrimitiveType::TYPE_SMALLINT:
405
                RETURN_IF_ERROR(Impl::template vector<ColumnInt16>(
406
                        offset_column_data, nested_column, nested_null_map, res, null_map));
407
                break;
408
            case PrimitiveType::TYPE_INT:
409
                RETURN_IF_ERROR(Impl::template vector<ColumnInt32>(
410
                        offset_column_data, nested_column, nested_null_map, res, null_map));
411
                break;
412
            case PrimitiveType::TYPE_BIGINT:
413
                RETURN_IF_ERROR(Impl::template vector<ColumnInt64>(
414
                        offset_column_data, nested_column, nested_null_map, res, null_map));
415
                break;
416
            default:
417
                return Status::RuntimeError("Illegal column {} of argument of function {}",
418
                                            block.get_by_position(arguments[0]).column->get_name(),
419
                                            get_name());
420
            }
421
        } else {
422
            return Status::RuntimeError("Illegal column {} of argument of function {}",
423
                                        block.get_by_position(arguments[0]).column->get_name(),
424
                                        get_name());
425
        }
426
24
        block.get_by_position(result).column =
427
25
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
428
25
        return Status::OK();
429
25
    }
_ZNK5doris24FunctionBitmapAlwaysNullINS_15BitmapFromArrayEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
372
13
                        uint32_t result, size_t input_rows_count) const override {
373
13
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
374
13
        auto res_data_column = ColumnBitmap::create();
375
13
        auto& null_map = res_null_map->get_data();
376
13
        auto& res = res_data_column->get_data();
377
378
13
        ColumnPtr& argument_column = block.get_by_position(arguments[0]).column;
379
        if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeString>) {
380
            const auto& str_column = static_cast<const ColumnString&>(*argument_column);
381
            const ColumnString::Chars& data = str_column.get_chars();
382
            const ColumnString::Offsets& offsets = str_column.get_offsets();
383
            RETURN_IF_ERROR(Impl::vector(data, offsets, res, null_map, input_rows_count));
384
13
        } else if constexpr (std::is_same_v<typename Impl::ArgumentType, DataTypeArray>) {
385
13
            auto argument_type = remove_nullable(
386
13
                    assert_cast<const DataTypeArray&>(*block.get_by_position(arguments[0]).type)
387
13
                            .get_nested_type());
388
13
            const auto& array_column = static_cast<const ColumnArray&>(*argument_column);
389
13
            const auto& offset_column_data = array_column.get_offsets();
390
13
            const auto& nested_nullable_column =
391
13
                    static_cast<const ColumnNullable&>(array_column.get_data());
392
13
            const auto& nested_column = nested_nullable_column.get_nested_column();
393
13
            const auto& nested_null_map = nested_nullable_column.get_null_map_column().get_data();
394
395
13
            switch (argument_type->get_primitive_type()) {
396
3
            case PrimitiveType::TYPE_TINYINT:
397
3
                RETURN_IF_ERROR(Impl::template vector<ColumnInt8>(offset_column_data, nested_column,
398
3
                                                                  nested_null_map, res, null_map));
399
3
                break;
400
3
            case PrimitiveType::TYPE_BOOLEAN:
401
0
                RETURN_IF_ERROR(Impl::template vector<ColumnUInt8>(
402
0
                        offset_column_data, nested_column, nested_null_map, res, null_map));
403
0
                break;
404
0
            case PrimitiveType::TYPE_SMALLINT:
405
0
                RETURN_IF_ERROR(Impl::template vector<ColumnInt16>(
406
0
                        offset_column_data, nested_column, nested_null_map, res, null_map));
407
0
                break;
408
3
            case PrimitiveType::TYPE_INT:
409
3
                RETURN_IF_ERROR(Impl::template vector<ColumnInt32>(
410
3
                        offset_column_data, nested_column, nested_null_map, res, null_map));
411
3
                break;
412
7
            case PrimitiveType::TYPE_BIGINT:
413
7
                RETURN_IF_ERROR(Impl::template vector<ColumnInt64>(
414
7
                        offset_column_data, nested_column, nested_null_map, res, null_map));
415
7
                break;
416
7
            default:
417
0
                return Status::RuntimeError("Illegal column {} of argument of function {}",
418
0
                                            block.get_by_position(arguments[0]).column->get_name(),
419
0
                                            get_name());
420
13
            }
421
        } else {
422
            return Status::RuntimeError("Illegal column {} of argument of function {}",
423
                                        block.get_by_position(arguments[0]).column->get_name(),
424
                                        get_name());
425
        }
426
13
        block.get_by_position(result).column =
427
13
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
428
13
        return Status::OK();
429
13
    }
430
};
431
432
template <int HashBits>
433
struct BitmapHashName {};
434
435
template <>
436
struct BitmapHashName<32> {
437
    static constexpr auto name = "bitmap_hash";
438
};
439
440
template <>
441
struct BitmapHashName<64> {
442
    static constexpr auto name = "bitmap_hash64";
443
};
444
445
template <int HashBits>
446
struct BitmapHash {
447
    static constexpr auto name = BitmapHashName<HashBits>::name;
448
449
    using ReturnType = DataTypeBitMap;
450
451
    template <typename ColumnType>
452
130
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
453
130
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
454
130
            const ColumnString::Chars& data = col->get_chars();
455
130
            const ColumnString::Offsets& offsets = col->get_offsets();
456
130
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
457
130
            auto& res_data = res_column->get_data();
458
130
            size_t size = offsets.size();
459
460
436
            for (size_t i = 0; i < size; ++i) {
461
306
                const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
462
306
                size_t str_size = offsets[i] - offsets[i - 1];
463
306
                if constexpr (HashBits == 32) {
464
274
                    uint32_t hash_value =
465
274
                            HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED);
466
274
                    res_data[i].add(hash_value);
467
274
                } else {
468
32
                    uint64_t hash_value = 0;
469
32
                    murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
470
32
                    res_data[i].add(hash_value);
471
32
                }
472
306
            }
473
130
        }
474
130
    }
_ZN5doris10BitmapHashILi32EE6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE
Line
Count
Source
452
114
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
453
114
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
454
114
            const ColumnString::Chars& data = col->get_chars();
455
114
            const ColumnString::Offsets& offsets = col->get_offsets();
456
114
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
457
114
            auto& res_data = res_column->get_data();
458
114
            size_t size = offsets.size();
459
460
388
            for (size_t i = 0; i < size; ++i) {
461
274
                const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
462
274
                size_t str_size = offsets[i] - offsets[i - 1];
463
274
                if constexpr (HashBits == 32) {
464
274
                    uint32_t hash_value =
465
274
                            HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED);
466
274
                    res_data[i].add(hash_value);
467
                } else {
468
                    uint64_t hash_value = 0;
469
                    murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
470
                    res_data[i].add(hash_value);
471
                }
472
274
            }
473
114
        }
474
114
    }
Unexecuted instantiation: _ZN5doris10BitmapHashILi32EE6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE
_ZN5doris10BitmapHashILi64EE6vectorINS_9ColumnStrIjEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrIS9_EE
Line
Count
Source
452
16
    static void vector(const ColumnType* col, MutableColumnPtr& col_res) {
453
16
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
454
16
            const ColumnString::Chars& data = col->get_chars();
455
16
            const ColumnString::Offsets& offsets = col->get_offsets();
456
16
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
457
16
            auto& res_data = res_column->get_data();
458
16
            size_t size = offsets.size();
459
460
48
            for (size_t i = 0; i < size; ++i) {
461
32
                const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
462
32
                size_t str_size = offsets[i] - offsets[i - 1];
463
                if constexpr (HashBits == 32) {
464
                    uint32_t hash_value =
465
                            HashUtil::murmur_hash3_32(raw_str, str_size, HashUtil::MURMUR3_32_SEED);
466
                    res_data[i].add(hash_value);
467
32
                } else {
468
32
                    uint64_t hash_value = 0;
469
32
                    murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
470
32
                    res_data[i].add(hash_value);
471
32
                }
472
32
            }
473
16
        }
474
16
    }
Unexecuted instantiation: _ZN5doris10BitmapHashILi64EE6vectorINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RNS_3COWINS_7IColumnEE11mutable_ptrISA_EE
475
476
    template <typename ColumnType>
477
    static void vector_nullable(const ColumnType* col, const NullMap& nullmap,
478
192
                                MutableColumnPtr& col_res) {
479
192
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
480
192
            const ColumnString::Chars& data = col->get_chars();
481
192
            const ColumnString::Offsets& offsets = col->get_offsets();
482
192
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
483
192
            auto& res_data = res_column->get_data();
484
192
            size_t size = offsets.size();
485
486
991
            for (size_t i = 0; i < size; ++i) {
487
799
                if (nullmap[i]) {
488
57
                    continue;
489
742
                } else {
490
742
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
491
742
                    size_t str_size = offsets[i] - offsets[i - 1];
492
742
                    if constexpr (HashBits == 32) {
493
716
                        uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size,
494
716
                                                                        HashUtil::MURMUR3_32_SEED);
495
716
                        res_data[i].add(hash_value);
496
716
                    } else {
497
26
                        uint64_t hash_value = 0;
498
26
                        murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
499
26
                        res_data[i].add(hash_value);
500
26
                    }
501
742
                }
502
799
            }
503
192
        }
504
192
    }
_ZN5doris10BitmapHashILi32EE15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
Line
Count
Source
478
183
                                MutableColumnPtr& col_res) {
479
183
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
480
183
            const ColumnString::Chars& data = col->get_chars();
481
183
            const ColumnString::Offsets& offsets = col->get_offsets();
482
183
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
483
183
            auto& res_data = res_column->get_data();
484
183
            size_t size = offsets.size();
485
486
955
            for (size_t i = 0; i < size; ++i) {
487
772
                if (nullmap[i]) {
488
56
                    continue;
489
716
                } else {
490
716
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
491
716
                    size_t str_size = offsets[i] - offsets[i - 1];
492
716
                    if constexpr (HashBits == 32) {
493
716
                        uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size,
494
716
                                                                        HashUtil::MURMUR3_32_SEED);
495
716
                        res_data[i].add(hash_value);
496
                    } else {
497
                        uint64_t hash_value = 0;
498
                        murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
499
                        res_data[i].add(hash_value);
500
                    }
501
716
                }
502
772
            }
503
183
        }
504
183
    }
Unexecuted instantiation: _ZN5doris10BitmapHashILi32EE15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE
_ZN5doris10BitmapHashILi64EE15vector_nullableINS_9ColumnStrIjEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISG_EE
Line
Count
Source
478
9
                                MutableColumnPtr& col_res) {
479
9
        if constexpr (std::is_same_v<ColumnType, ColumnString>) {
480
9
            const ColumnString::Chars& data = col->get_chars();
481
9
            const ColumnString::Offsets& offsets = col->get_offsets();
482
9
            auto* res_column = reinterpret_cast<ColumnBitmap*>(col_res.get());
483
9
            auto& res_data = res_column->get_data();
484
9
            size_t size = offsets.size();
485
486
36
            for (size_t i = 0; i < size; ++i) {
487
27
                if (nullmap[i]) {
488
1
                    continue;
489
26
                } else {
490
26
                    const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
491
26
                    size_t str_size = offsets[i] - offsets[i - 1];
492
                    if constexpr (HashBits == 32) {
493
                        uint32_t hash_value = HashUtil::murmur_hash3_32(raw_str, str_size,
494
                                                                        HashUtil::MURMUR3_32_SEED);
495
                        res_data[i].add(hash_value);
496
26
                    } else {
497
26
                        uint64_t hash_value = 0;
498
26
                        murmur_hash3_x64_64(raw_str, str_size, 0, &hash_value);
499
26
                        res_data[i].add(hash_value);
500
26
                    }
501
26
                }
502
27
            }
503
9
        }
504
9
    }
Unexecuted instantiation: _ZN5doris10BitmapHashILi64EE15vector_nullableINS_12ColumnVectorILNS_13PrimitiveTypeE6EEEEEvPKT_RKNS_8PODArrayIhLm4096ENS_9AllocatorILb0ELb0ELb0ENS_22DefaultMemoryAllocatorELb1EEELm16ELm15EEERNS_3COWINS_7IColumnEE11mutable_ptrISH_EE
505
};
506
507
class FunctionBitmapCount : public IFunction {
508
public:
509
    static constexpr auto name = "bitmap_count";
510
511
1
    String get_name() const override { return name; }
512
513
80
    static FunctionPtr create() { return std::make_shared<FunctionBitmapCount>(); }
514
515
71
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
516
71
        return std::make_shared<DataTypeInt64>();
517
71
    }
518
519
71
    size_t get_number_of_arguments() const override { return 1; }
520
521
188
    bool use_default_implementation_for_nulls() const override { return false; }
522
523
    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
524
116
                        uint32_t result, size_t input_rows_count) const override {
525
116
        auto res_data_column = ColumnInt64::create();
526
116
        auto& res = res_data_column->get_data();
527
116
        auto data_null_map = ColumnUInt8::create(input_rows_count, 0);
528
116
        auto& null_map = data_null_map->get_data();
529
530
116
        auto column = block.get_by_position(arguments[0]).column;
531
116
        if (auto* nullable = check_and_get_column<const ColumnNullable>(*column)) {
532
19
            VectorizedUtils::update_null_map(null_map, nullable->get_null_map_data());
533
19
            column = nullable->get_nested_column_ptr();
534
19
        }
535
116
        auto str_col = assert_cast<const ColumnBitmap*>(column.get());
536
116
        const auto& col_data = str_col->get_data();
537
538
116
        res.reserve(input_rows_count);
539
321
        for (size_t i = 0; i < input_rows_count; ++i) {
540
205
            if (null_map[i]) {
541
3
                res.push_back(0);
542
3
                continue;
543
3
            }
544
202
            res.push_back(col_data[i].cardinality());
545
202
        }
546
116
        block.replace_by_position(result, std::move(res_data_column));
547
116
        return Status::OK();
548
116
    }
549
};
550
551
struct NameBitmapNot {
552
    static constexpr auto name = "bitmap_not";
553
};
554
555
template <typename LeftDataType, typename RightDataType>
556
struct BitmapNot {
557
    using ResultDataType = DataTypeBitMap;
558
    using T0 = typename LeftDataType::FieldType;
559
    using T1 = typename RightDataType::FieldType;
560
    using TData = std::vector<BitmapValue>;
561
562
10
    static void vector_vector(const TData& lvec, const TData& rvec, TData& res) {
563
10
        size_t size = lvec.size();
564
37
        for (size_t i = 0; i < size; ++i) {
565
27
            res[i] = lvec[i];
566
27
            res[i] -= rvec[i];
567
27
        }
568
10
    }
569
0
    static void vector_scalar(const TData& lvec, const BitmapValue& rval, TData& res) {
570
0
        size_t size = lvec.size();
571
0
        for (size_t i = 0; i < size; ++i) {
572
0
            res[i] = lvec[i];
573
0
            res[i] -= rval;
574
0
        }
575
0
    }
576
0
    static void scalar_vector(const BitmapValue& lval, const TData& rvec, TData& res) {
577
0
        size_t size = rvec.size();
578
0
        for (size_t i = 0; i < size; ++i) {
579
0
            res[i] = lval;
580
0
            res[i] -= rvec[i];
581
0
        }
582
0
    }
583
};
584
585
struct NameBitmapAndNot {
586
    static constexpr auto name = "bitmap_and_not";
587
};
588
589
template <typename LeftDataType, typename RightDataType>
590
struct BitmapAndNot {
591
    using ResultDataType = DataTypeBitMap;
592
    using T0 = typename LeftDataType::FieldType;
593
    using T1 = typename RightDataType::FieldType;
594
    using TData = std::vector<BitmapValue>;
595
596
19
    static void vector_vector(const TData& lvec, const TData& rvec, TData& res) {
597
19
        size_t size = lvec.size();
598
19
        BitmapValue mid_data;
599
74
        for (size_t i = 0; i < size; ++i) {
600
55
            mid_data = lvec[i];
601
55
            mid_data &= rvec[i];
602
55
            res[i] = lvec[i];
603
55
            res[i] -= mid_data;
604
55
            mid_data.reset();
605
55
        }
606
19
    }
607
0
    static void vector_scalar(const TData& lvec, const BitmapValue& rval, TData& res) {
608
0
        size_t size = lvec.size();
609
0
        BitmapValue mid_data;
610
0
        for (size_t i = 0; i < size; ++i) {
611
0
            mid_data = lvec[i];
612
0
            mid_data &= rval;
613
0
            res[i] = lvec[i];
614
0
            res[i] -= mid_data;
615
0
            mid_data.reset();
616
0
        }
617
0
    }
618
0
    static void scalar_vector(const BitmapValue& lval, const TData& rvec, TData& res) {
619
0
        size_t size = rvec.size();
620
0
        BitmapValue mid_data;
621
0
        for (size_t i = 0; i < size; ++i) {
622
0
            mid_data = lval;
623
0
            mid_data &= rvec[i];
624
0
            res[i] = lval;
625
0
            res[i] -= mid_data;
626
0
            mid_data.reset();
627
0
        }
628
0
    }
629
};
630
631
struct NameBitmapAndNotCount {
632
    static constexpr auto name = "bitmap_and_not_count";
633
};
634
635
template <typename LeftDataType, typename RightDataType>
636
struct BitmapAndNotCount {
637
    using ResultDataType = DataTypeInt64;
638
    using T0 = typename LeftDataType::FieldType;
639
    using T1 = typename RightDataType::FieldType;
640
    using TData = std::vector<BitmapValue>;
641
    using ResTData = typename ColumnInt64::Container::value_type;
642
643
29
    static void vector_vector(const TData& lvec, const TData& rvec, ResTData* res) {
644
29
        size_t size = lvec.size();
645
29
        BitmapValue mid_data;
646
107
        for (size_t i = 0; i < size; ++i) {
647
78
            mid_data = lvec[i];
648
78
            mid_data &= rvec[i];
649
78
            res[i] = lvec[i].andnot_cardinality(mid_data);
650
78
            mid_data.reset();
651
78
        }
652
29
    }
653
0
    static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData* res) {
654
0
        size_t size = rvec.size();
655
0
        BitmapValue mid_data;
656
0
        for (size_t i = 0; i < size; ++i) {
657
0
            mid_data = lval;
658
0
            mid_data &= rvec[i];
659
0
            res[i] = lval.andnot_cardinality(mid_data);
660
0
            mid_data.reset();
661
0
        }
662
0
    }
663
0
    static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData* res) {
664
0
        size_t size = lvec.size();
665
0
        BitmapValue mid_data;
666
0
        for (size_t i = 0; i < size; ++i) {
667
0
            mid_data = lvec[i];
668
0
            mid_data &= rval;
669
0
            res[i] = lvec[i].andnot_cardinality(mid_data);
670
0
            mid_data.reset();
671
0
        }
672
0
    }
673
};
674
675
63
void update_bitmap_op_count(int64_t* __restrict count, const NullMap& null_map) {
676
63
    static constexpr int64_t flags[2] = {-1, 0};
677
63
    size_t size = null_map.size();
678
63
    auto* __restrict null_map_data = null_map.data();
679
178
    for (size_t i = 0; i < size; ++i) {
680
115
        count[i] &= flags[null_map_data[i]];
681
115
    }
682
63
}
683
684
// for bitmap_and_count, bitmap_xor_count and bitmap_and_not_count,
685
// result is 0 for rows that if any column is null value
686
ColumnPtr handle_bitmap_op_count_null_value(ColumnPtr& src, const Block& block,
687
                                            const ColumnNumbers& args, uint32_t result,
688
34
                                            size_t input_rows_count) {
689
34
    MutableColumnPtr mutable_src = IColumn::mutate(std::move(src));
690
34
    auto* nullable = assert_cast<ColumnNullable*>(mutable_src.get());
691
34
    auto* src_not_nullable_mutable = &nullable->get_nested_column();
692
34
    auto* __restrict count_data =
693
34
            assert_cast<ColumnInt64*>(src_not_nullable_mutable)->get_data().data();
694
695
80
    for (const auto& arg : args) {
696
80
        const ColumnWithTypeAndName& elem = block.get_by_position(arg);
697
80
        if (!elem.type->is_nullable()) {
698
17
            continue;
699
17
        }
700
701
63
        bool is_const = is_column_const(*elem.column);
702
        /// Const Nullable that are NULL.
703
63
        if (is_const && assert_cast<const ColumnConst*>(elem.column.get())->only_null()) {
704
0
            return block.get_by_position(result).type->create_column_const(
705
0
                    input_rows_count, Field::create_field<TYPE_BIGINT>(0));
706
0
        }
707
63
        if (is_const) {
708
0
            continue;
709
0
        }
710
711
63
        if (const auto* nullable_column = assert_cast<const ColumnNullable*>(elem.column.get())) {
712
63
            const ColumnPtr& null_map_column = nullable_column->get_null_map_column_ptr();
713
63
            const NullMap& src_null_map =
714
63
                    assert_cast<const ColumnUInt8&>(*null_map_column).get_data();
715
716
63
            update_bitmap_op_count(count_data, src_null_map);
717
63
        }
718
63
    }
719
720
34
    return mutable_src;
721
34
}
722
723
Status execute_bitmap_op_count_null_to_zero(
724
        FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result,
725
        size_t input_rows_count,
726
        const std::function<Status(FunctionContext*, Block&, const ColumnNumbers&, size_t, size_t)>&
727
75
                exec_impl_func) {
728
75
    if (have_null_column(block, arguments)) {
729
34
        auto [temporary_block, new_args, new_result] =
730
34
                create_block_with_nested_columns(block, arguments, result);
731
34
        RETURN_IF_ERROR(exec_impl_func(context, temporary_block, new_args, new_result,
732
34
                                       temporary_block.rows()));
733
34
        block.get_by_position(result).column = handle_bitmap_op_count_null_value(
734
34
                temporary_block.get_by_position(new_result).column, block, arguments, result,
735
34
                input_rows_count);
736
41
    } else {
737
41
        return exec_impl_func(context, block, arguments, result, input_rows_count);
738
41
    }
739
34
    return Status::OK();
740
75
}
741
742
template <typename FunctionName>
743
class FunctionBitmapAndNotCount : public IFunction {
744
public:
745
    using LeftDataType = DataTypeBitMap;
746
    using RightDataType = DataTypeBitMap;
747
    using ResultDataType = typename BitmapAndNotCount<LeftDataType, RightDataType>::ResultDataType;
748
749
    static constexpr auto name = FunctionName::name;
750
23
    static FunctionPtr create() { return std::make_shared<FunctionBitmapAndNotCount>(); }
751
1
    String get_name() const override { return name; }
752
14
    size_t get_number_of_arguments() const override { return 2; }
753
14
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
754
14
        bool return_nullable = false;
755
        // result is nullable only when any columns is nullable for bitmap_and_not_count
756
31
        for (size_t i = 0; i < arguments.size(); ++i) {
757
24
            if (arguments[i]->is_nullable()) {
758
7
                return_nullable = true;
759
7
                break;
760
7
            }
761
24
        }
762
14
        auto result_type = std::make_shared<ResultDataType>();
763
14
        return return_nullable ? make_nullable(result_type) : result_type;
764
14
    }
765
766
43
    bool use_default_implementation_for_nulls() const override {
767
        // for bitmap_and_not_count, result is always not null, and if the bitmap op result is null,
768
        // the count is 0
769
43
        return false;
770
43
    }
771
772
    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
773
29
                        uint32_t result, size_t input_rows_count) const override {
774
29
        DCHECK_EQ(arguments.size(), 2);
775
29
        auto impl_func = [&](FunctionContext* context, Block& block, const ColumnNumbers& arguments,
776
29
                             uint32_t result, size_t input_rows_count) {
777
29
            return execute_impl_internal(context, block, arguments, result, input_rows_count);
778
29
        };
779
29
        return execute_bitmap_op_count_null_to_zero(context, block, arguments, result,
780
29
                                                    input_rows_count, impl_func);
781
29
    }
782
783
    Status execute_impl_internal(FunctionContext* context, Block& block,
784
                                 const ColumnNumbers& arguments, uint32_t result,
785
29
                                 size_t input_rows_count) const {
786
29
        using ColVecResult = ColumnVector<ResultDataType::PType>;
787
788
29
        typename ColVecResult::MutablePtr col_res = ColVecResult::create();
789
29
        auto& vec_res = col_res->get_data();
790
29
        vec_res.resize(block.rows());
791
792
29
        const auto& left = block.get_by_position(arguments[0]);
793
29
        auto lcol = left.column;
794
29
        const auto& right = block.get_by_position(arguments[1]);
795
29
        auto rcol = right.column;
796
797
29
        if (is_column_const(*left.column)) {
798
0
            BitmapAndNotCount<LeftDataType, RightDataType>::scalar_vector(
799
0
                    assert_cast<const ColumnBitmap&>(
800
0
                            assert_cast<const ColumnConst*>(lcol.get())->get_data_column())
801
0
                            .get_data()[0],
802
0
                    assert_cast<const ColumnBitmap*>(rcol.get())->get_data(), vec_res.data());
803
29
        } else if (is_column_const(*right.column)) {
804
0
            BitmapAndNotCount<LeftDataType, RightDataType>::vector_scalar(
805
0
                    assert_cast<const ColumnBitmap*>(lcol.get())->get_data(),
806
0
                    assert_cast<const ColumnBitmap&>(
807
0
                            assert_cast<const ColumnConst*>(rcol.get())->get_data_column())
808
0
                            .get_data()[0],
809
0
                    vec_res.data());
810
29
        } else {
811
29
            BitmapAndNotCount<LeftDataType, RightDataType>::vector_vector(
812
29
                    assert_cast<const ColumnBitmap*>(lcol.get())->get_data(),
813
29
                    assert_cast<const ColumnBitmap*>(rcol.get())->get_data(), vec_res.data());
814
29
        }
815
816
29
        auto& result_info = block.get_by_position(result);
817
29
        if (result_info.type->is_nullable()) {
818
10
            block.replace_by_position(
819
10
                    result, ColumnNullable::create(std::move(col_res),
820
10
                                                   ColumnUInt8::create(input_rows_count, 0)));
821
19
        } else {
822
19
            block.replace_by_position(result, std::move(col_res));
823
19
        }
824
29
        return Status::OK();
825
29
    }
826
};
827
828
struct NameBitmapContains {
829
    static constexpr auto name = "bitmap_contains";
830
};
831
832
template <typename LeftDataType, typename RightDataType>
833
struct BitmapContains {
834
    using ResultDataType = DataTypeUInt8;
835
    using T0 = typename LeftDataType::FieldType;
836
    using T1 = typename RightDataType::FieldType;
837
    using LTData = std::vector<BitmapValue>;
838
    using RTData = typename ColumnVector<RightDataType::PType>::Container;
839
    using ResTData = typename ColumnUInt8::Container;
840
841
10
    static void vector_vector(const LTData& lvec, const RTData& rvec, ResTData& res) {
842
10
        size_t size = lvec.size();
843
37
        for (size_t i = 0; i < size; ++i) {
844
27
            res[i] = lvec[i].contains(rvec[i]);
845
27
        }
846
10
    }
847
410
    static void vector_scalar(const LTData& lvec, const T1& rval, ResTData& res) {
848
410
        size_t size = lvec.size();
849
820
        for (size_t i = 0; i < size; ++i) {
850
410
            res[i] = lvec[i].contains(rval);
851
410
        }
852
410
    }
853
0
    static void scalar_vector(const BitmapValue& lval, const RTData& rvec, ResTData& res) {
854
0
        size_t size = rvec.size();
855
0
        for (size_t i = 0; i < size; ++i) {
856
0
            res[i] = lval.contains(rvec[i]);
857
0
        }
858
0
    }
859
};
860
861
struct NameBitmapRemove {
862
    static constexpr auto name = "bitmap_remove";
863
};
864
865
template <typename LeftDataType, typename RightDataType>
866
struct BitmapRemove {
867
    using ResultDataType = DataTypeBitMap;
868
    using T0 = typename LeftDataType::FieldType;
869
    using T1 = typename RightDataType::FieldType;
870
    using LTData = std::vector<BitmapValue>;
871
    using RTData = typename ColumnVector<RightDataType::PType>::Container;
872
    using ResTData = std::vector<BitmapValue>;
873
874
2
    static void vector_vector(const LTData& lvec, const RTData& rvec, ResTData& res) {
875
2
        size_t size = lvec.size();
876
6
        for (size_t i = 0; i < size; ++i) {
877
4
            res[i] = lvec[i];
878
4
            res[i].remove(rvec[i]);
879
4
        }
880
2
    }
881
12
    static void vector_scalar(const LTData& lvec, const T1& rval, ResTData& res) {
882
12
        size_t size = lvec.size();
883
45
        for (size_t i = 0; i < size; ++i) {
884
33
            res[i] = lvec[i];
885
33
            res[i].remove(rval);
886
33
        }
887
12
    }
888
0
    static void scalar_vector(const BitmapValue& lval, const RTData& rvec, ResTData& res) {
889
0
        size_t size = rvec.size();
890
0
        for (size_t i = 0; i < size; ++i) {
891
0
            res[i] = lval;
892
0
            res[i].remove(rvec[i]);
893
0
        }
894
0
    }
895
};
896
897
struct NameBitmapHasAny {
898
    static constexpr auto name = "bitmap_has_any";
899
};
900
901
template <typename LeftDataType, typename RightDataType>
902
struct BitmapHasAny {
903
    using ResultDataType = DataTypeUInt8;
904
    using T0 = typename LeftDataType::FieldType;
905
    using T1 = typename RightDataType::FieldType;
906
    using TData = std::vector<BitmapValue>;
907
    using ResTData = typename ColumnUInt8::Container;
908
909
0
    static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) {
910
0
        size_t size = lvec.size();
911
0
        for (size_t i = 0; i < size; ++i) {
912
0
            res[i] = lvec[i].intersects(rvec[i]);
913
0
        }
914
0
    }
915
0
    static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) {
916
0
        size_t size = lvec.size();
917
0
        for (size_t i = 0; i < size; ++i) {
918
0
            res[i] = lvec[i].intersects(rval);
919
0
        }
920
0
    }
921
0
    static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) {
922
0
        size_t size = rvec.size();
923
0
        for (size_t i = 0; i < size; ++i) {
924
0
            res[i] = lval.intersects(rvec[i]);
925
0
        }
926
0
    }
927
};
928
929
struct NameBitmapHasAll {
930
    static constexpr auto name = "bitmap_has_all";
931
};
932
933
template <typename LeftDataType, typename RightDataType>
934
struct BitmapHasAll {
935
    using ResultDataType = DataTypeUInt8;
936
    using T0 = typename LeftDataType::FieldType;
937
    using T1 = typename RightDataType::FieldType;
938
    using TData = std::vector<BitmapValue>;
939
    using ResTData = typename ColumnUInt8::Container;
940
941
1
    static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) {
942
1
        size_t size = lvec.size();
943
6
        for (size_t i = 0; i < size; ++i) {
944
5
            res[i] = lvec[i].contains_all(rvec[i]);
945
5
        }
946
1
    }
947
0
    static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) {
948
0
        size_t size = lvec.size();
949
0
        for (size_t i = 0; i < size; ++i) {
950
0
            res[i] = lvec[i].contains_all(rval);
951
0
        }
952
0
    }
953
0
    static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) {
954
0
        size_t size = rvec.size();
955
0
        for (size_t i = 0; i < size; ++i) {
956
0
            res[i] = lval.contains_all(rvec[i]);
957
0
        }
958
0
    }
959
};
960
961
struct NameBitmapToString {
962
    static constexpr auto name = "bitmap_to_string";
963
};
964
965
struct BitmapToString {
966
    using ReturnType = DataTypeString;
967
    static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_BITMAP;
968
    using Type = DataTypeBitMap::FieldType;
969
    using ReturnColumnType = ColumnString;
970
    using Chars = ColumnString::Chars;
971
    using Offsets = ColumnString::Offsets;
972
973
236
    static Status vector(const std::vector<BitmapValue>& data, Chars& chars, Offsets& offsets) {
974
236
        size_t size = data.size();
975
236
        offsets.resize(size);
976
236
        chars.reserve(size);
977
1.20k
        for (size_t i = 0; i < size; ++i) {
978
967
            StringOP::push_value_string(data[i].to_string(), i, chars, offsets);
979
967
        }
980
236
        return Status::OK();
981
236
    }
982
};
983
984
struct NameBitmapToBase64 {
985
    static constexpr auto name = "bitmap_to_base64";
986
};
987
988
struct BitmapToBase64 {
989
    using ReturnType = DataTypeString;
990
    static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_BITMAP;
991
    using Type = DataTypeBitMap::FieldType;
992
    using ReturnColumnType = ColumnString;
993
    using Chars = ColumnString::Chars;
994
    using Offsets = ColumnString::Offsets;
995
996
    // ColumnString not support 64bit, only 32bit, so that the max size is 4G
997
25
    static Status vector(const std::vector<BitmapValue>& data, Chars& chars, Offsets& offsets) {
998
25
        size_t size = data.size();
999
25
        offsets.resize(size);
1000
25
        size_t output_char_size = 0;
1001
71
        for (size_t i = 0; i < size; ++i) {
1002
46
            const BitmapValue& bitmap_val = data[i];
1003
46
            auto ser_size = bitmap_val.getSizeInBytes();
1004
46
            output_char_size += (int)(4.0 * ceil((double)ser_size / 3.0));
1005
46
        }
1006
25
        ColumnString::check_chars_length(output_char_size, size);
1007
25
        chars.resize(output_char_size);
1008
25
        auto chars_data = chars.data();
1009
1010
25
        size_t cur_ser_size = 0;
1011
25
        size_t last_ser_size = 0;
1012
25
        std::string ser_buff;
1013
25
        size_t encoded_offset = 0;
1014
71
        for (size_t i = 0; i < size; ++i) {
1015
46
            const BitmapValue& bitmap_val = data[i];
1016
46
            cur_ser_size = bitmap_val.getSizeInBytes();
1017
46
            if (cur_ser_size > last_ser_size) {
1018
35
                last_ser_size = cur_ser_size;
1019
35
                ser_buff.resize(cur_ser_size);
1020
35
            }
1021
46
            bitmap_val.write_to(ser_buff.data());
1022
1023
46
            auto outlen = base64_encode((const unsigned char*)ser_buff.data(), cur_ser_size,
1024
46
                                        chars_data + encoded_offset);
1025
46
            DCHECK(outlen > 0);
1026
1027
46
            encoded_offset += (int)(4.0 * ceil((double)cur_ser_size / 3.0));
1028
46
            offsets[i] = cast_set<UInt32>(encoded_offset);
1029
46
        }
1030
25
        return Status::OK();
1031
25
    }
1032
};
1033
1034
struct SubBitmap {
1035
    static constexpr auto name = "sub_bitmap";
1036
    using TData1 = std::vector<BitmapValue>;
1037
    using TData2 = typename ColumnInt64::Container;
1038
1039
    static void vector3(const TData1& bitmap_data, const TData2& offset_data,
1040
                        const TData2& limit_data, NullMap& null_map, size_t input_rows_count,
1041
13
                        TData1& res) {
1042
43
        for (int i = 0; i < input_rows_count; ++i) {
1043
30
            if (null_map[i]) {
1044
0
                continue;
1045
0
            }
1046
30
            if (limit_data[i] <= 0) {
1047
1
                null_map[i] = 1;
1048
1
                continue;
1049
1
            }
1050
29
            if (bitmap_data[i].offset_limit(offset_data[i], limit_data[i], &res[i]) == 0) {
1051
24
                null_map[i] = 1;
1052
24
            }
1053
29
        }
1054
13
    }
1055
    static void vector_scalars(const TData1& bitmap_data, const Int64& offset_data,
1056
                               const Int64& limit_data, NullMap& null_map, size_t input_rows_count,
1057
0
                               TData1& res) {
1058
0
        for (int i = 0; i < input_rows_count; ++i) {
1059
0
            if (null_map[i]) {
1060
0
                continue;
1061
0
            }
1062
0
            if (limit_data <= 0) {
1063
0
                null_map[i] = 1;
1064
0
                continue;
1065
0
            }
1066
0
            if (bitmap_data[i].offset_limit(offset_data, limit_data, &res[i]) == 0) {
1067
0
                null_map[i] = 1;
1068
0
            }
1069
0
        }
1070
0
    }
1071
};
1072
1073
struct BitmapSubsetLimit {
1074
    static constexpr auto name = "bitmap_subset_limit";
1075
    using TData1 = std::vector<BitmapValue>;
1076
    using TData2 = typename ColumnInt64::Container;
1077
1078
    static void vector3(const TData1& bitmap_data, const TData2& offset_data,
1079
                        const TData2& limit_data, NullMap& null_map, size_t input_rows_count,
1080
11
                        TData1& res) {
1081
39
        for (int i = 0; i < input_rows_count; ++i) {
1082
28
            if (null_map[i]) {
1083
0
                continue;
1084
0
            }
1085
28
            if (offset_data[i] < 0 || limit_data[i] < 0) {
1086
0
                null_map[i] = 1;
1087
0
                continue;
1088
0
            }
1089
28
            bitmap_data[i].sub_limit(offset_data[i], limit_data[i], &res[i]);
1090
28
        }
1091
11
    }
1092
    static void vector_scalars(const TData1& bitmap_data, const Int64& offset_data,
1093
                               const Int64& limit_data, NullMap& null_map, size_t input_rows_count,
1094
0
                               TData1& res) {
1095
0
        for (int i = 0; i < input_rows_count; ++i) {
1096
0
            if (null_map[i]) {
1097
0
                continue;
1098
0
            }
1099
0
            if (offset_data < 0 || limit_data < 0) {
1100
0
                null_map[i] = 1;
1101
0
                continue;
1102
0
            }
1103
0
            bitmap_data[i].sub_limit(offset_data, limit_data, &res[i]);
1104
0
        }
1105
0
    }
1106
};
1107
1108
struct BitmapSubsetInRange {
1109
    static constexpr auto name = "bitmap_subset_in_range";
1110
    using TData1 = std::vector<BitmapValue>;
1111
    using TData2 = typename ColumnInt64::Container;
1112
1113
    static void vector3(const TData1& bitmap_data, const TData2& range_start,
1114
                        const TData2& range_end, NullMap& null_map, size_t input_rows_count,
1115
11
                        TData1& res) {
1116
39
        for (int i = 0; i < input_rows_count; ++i) {
1117
28
            if (null_map[i]) {
1118
0
                continue;
1119
0
            }
1120
28
            if (range_start[i] >= range_end[i] || range_start[i] < 0 || range_end[i] < 0) {
1121
25
                null_map[i] = 1;
1122
25
                continue;
1123
25
            }
1124
3
            bitmap_data[i].sub_range(range_start[i], range_end[i], &res[i]);
1125
3
        }
1126
11
    }
1127
    static void vector_scalars(const TData1& bitmap_data, const Int64& range_start,
1128
                               const Int64& range_end, NullMap& null_map, size_t input_rows_count,
1129
0
                               TData1& res) {
1130
0
        for (int i = 0; i < input_rows_count; ++i) {
1131
0
            if (null_map[i]) {
1132
0
                continue;
1133
0
            }
1134
0
            if (range_start >= range_end || range_start < 0 || range_end < 0) {
1135
0
                null_map[i] = 1;
1136
0
                continue;
1137
0
            }
1138
0
            bitmap_data[i].sub_range(range_start, range_end, &res[i]);
1139
0
        }
1140
0
    }
1141
};
1142
1143
template <typename Impl>
1144
class FunctionBitmapSubs : public IFunction {
1145
public:
1146
    static constexpr auto name = Impl::name;
1147
3
    String get_name() const override { return name; }
_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE8get_nameB5cxx11Ev
Line
Count
Source
1147
1
    String get_name() const override { return name; }
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE8get_nameB5cxx11Ev
Line
Count
Source
1147
1
    String get_name() const override { return name; }
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE8get_nameB5cxx11Ev
Line
Count
Source
1147
1
    String get_name() const override { return name; }
1148
1149
44
    static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); }
_ZN5doris18FunctionBitmapSubsINS_9SubBitmapEE6createEv
Line
Count
Source
1149
16
    static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); }
_ZN5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE6createEv
Line
Count
Source
1149
14
    static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); }
_ZN5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE6createEv
Line
Count
Source
1149
14
    static FunctionPtr create() { return std::make_shared<FunctionBitmapSubs>(); }
1150
1151
17
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
1152
17
        return make_nullable(std::make_shared<DataTypeBitMap>());
1153
17
    }
_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
1151
7
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
1152
7
        return make_nullable(std::make_shared<DataTypeBitMap>());
1153
7
    }
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
1151
5
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
1152
5
        return make_nullable(std::make_shared<DataTypeBitMap>());
1153
5
    }
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE20get_return_type_implERKSt6vectorISt10shared_ptrIKNS_9IDataTypeEESaIS7_EE
Line
Count
Source
1151
5
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
1152
5
        return make_nullable(std::make_shared<DataTypeBitMap>());
1153
5
    }
1154
1155
17
    size_t get_number_of_arguments() const override { return 3; }
_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE23get_number_of_argumentsEv
Line
Count
Source
1155
7
    size_t get_number_of_arguments() const override { return 3; }
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE23get_number_of_argumentsEv
Line
Count
Source
1155
5
    size_t get_number_of_arguments() const override { return 3; }
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE23get_number_of_argumentsEv
Line
Count
Source
1155
5
    size_t get_number_of_arguments() const override { return 3; }
1156
1157
    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
1158
34
                        uint32_t result, size_t input_rows_count) const override {
1159
34
        DCHECK_EQ(arguments.size(), 3);
1160
34
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
1161
34
        auto res_data_column = ColumnBitmap::create(input_rows_count);
1162
1163
34
        bool col_const[3];
1164
34
        ColumnPtr argument_columns[3];
1165
139
        for (int i = 0; i < 3; ++i) {
1166
105
            col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column);
1167
105
        }
1168
34
        argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>(
1169
0
                                                     *block.get_by_position(arguments[0]).column)
1170
0
                                                     .convert_to_full_column()
1171
34
                                           : block.get_by_position(arguments[0]).column;
1172
1173
34
        default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments);
1174
1175
34
        auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get());
1176
34
        auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get());
1177
34
        auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get());
1178
1179
34
        if (col_const[1] && col_const[2]) {
1180
0
            Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0),
1181
0
                                 limit_column->get_element(0), res_null_map->get_data(),
1182
0
                                 input_rows_count, res_data_column->get_data());
1183
34
        } else {
1184
34
            Impl::vector3(bitmap_column->get_data(), offset_column->get_data(),
1185
34
                          limit_column->get_data(), res_null_map->get_data(), input_rows_count,
1186
34
                          res_data_column->get_data());
1187
34
        }
1188
1189
34
        block.get_by_position(result).column =
1190
34
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
1191
34
        return Status::OK();
1192
34
    }
_ZNK5doris18FunctionBitmapSubsINS_9SubBitmapEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
1158
13
                        uint32_t result, size_t input_rows_count) const override {
1159
13
        DCHECK_EQ(arguments.size(), 3);
1160
13
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
1161
13
        auto res_data_column = ColumnBitmap::create(input_rows_count);
1162
1163
13
        bool col_const[3];
1164
13
        ColumnPtr argument_columns[3];
1165
52
        for (int i = 0; i < 3; ++i) {
1166
39
            col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column);
1167
39
        }
1168
13
        argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>(
1169
0
                                                     *block.get_by_position(arguments[0]).column)
1170
0
                                                     .convert_to_full_column()
1171
13
                                           : block.get_by_position(arguments[0]).column;
1172
1173
13
        default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments);
1174
1175
13
        auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get());
1176
13
        auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get());
1177
13
        auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get());
1178
1179
13
        if (col_const[1] && col_const[2]) {
1180
0
            Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0),
1181
0
                                 limit_column->get_element(0), res_null_map->get_data(),
1182
0
                                 input_rows_count, res_data_column->get_data());
1183
13
        } else {
1184
13
            Impl::vector3(bitmap_column->get_data(), offset_column->get_data(),
1185
13
                          limit_column->get_data(), res_null_map->get_data(), input_rows_count,
1186
13
                          res_data_column->get_data());
1187
13
        }
1188
1189
13
        block.get_by_position(result).column =
1190
13
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
1191
13
        return Status::OK();
1192
13
    }
_ZNK5doris18FunctionBitmapSubsINS_17BitmapSubsetLimitEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
1158
11
                        uint32_t result, size_t input_rows_count) const override {
1159
11
        DCHECK_EQ(arguments.size(), 3);
1160
11
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
1161
11
        auto res_data_column = ColumnBitmap::create(input_rows_count);
1162
1163
11
        bool col_const[3];
1164
11
        ColumnPtr argument_columns[3];
1165
44
        for (int i = 0; i < 3; ++i) {
1166
33
            col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column);
1167
33
        }
1168
11
        argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>(
1169
0
                                                     *block.get_by_position(arguments[0]).column)
1170
0
                                                     .convert_to_full_column()
1171
11
                                           : block.get_by_position(arguments[0]).column;
1172
1173
11
        default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments);
1174
1175
11
        auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get());
1176
11
        auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get());
1177
11
        auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get());
1178
1179
11
        if (col_const[1] && col_const[2]) {
1180
0
            Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0),
1181
0
                                 limit_column->get_element(0), res_null_map->get_data(),
1182
0
                                 input_rows_count, res_data_column->get_data());
1183
11
        } else {
1184
11
            Impl::vector3(bitmap_column->get_data(), offset_column->get_data(),
1185
11
                          limit_column->get_data(), res_null_map->get_data(), input_rows_count,
1186
11
                          res_data_column->get_data());
1187
11
        }
1188
1189
11
        block.get_by_position(result).column =
1190
11
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
1191
11
        return Status::OK();
1192
11
    }
_ZNK5doris18FunctionBitmapSubsINS_19BitmapSubsetInRangeEE12execute_implEPNS_15FunctionContextERNS_5BlockERKSt6vectorIjSaIjEEjm
Line
Count
Source
1158
10
                        uint32_t result, size_t input_rows_count) const override {
1159
10
        DCHECK_EQ(arguments.size(), 3);
1160
10
        auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
1161
10
        auto res_data_column = ColumnBitmap::create(input_rows_count);
1162
1163
10
        bool col_const[3];
1164
10
        ColumnPtr argument_columns[3];
1165
43
        for (int i = 0; i < 3; ++i) {
1166
33
            col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column);
1167
33
        }
1168
10
        argument_columns[0] = col_const[0] ? static_cast<const ColumnConst&>(
1169
0
                                                     *block.get_by_position(arguments[0]).column)
1170
0
                                                     .convert_to_full_column()
1171
10
                                           : block.get_by_position(arguments[0]).column;
1172
1173
10
        default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments);
1174
1175
10
        auto bitmap_column = assert_cast<const ColumnBitmap*>(argument_columns[0].get());
1176
10
        auto offset_column = assert_cast<const ColumnInt64*>(argument_columns[1].get());
1177
10
        auto limit_column = assert_cast<const ColumnInt64*>(argument_columns[2].get());
1178
1179
10
        if (col_const[1] && col_const[2]) {
1180
0
            Impl::vector_scalars(bitmap_column->get_data(), offset_column->get_element(0),
1181
0
                                 limit_column->get_element(0), res_null_map->get_data(),
1182
0
                                 input_rows_count, res_data_column->get_data());
1183
10
        } else {
1184
10
            Impl::vector3(bitmap_column->get_data(), offset_column->get_data(),
1185
10
                          limit_column->get_data(), res_null_map->get_data(), input_rows_count,
1186
10
                          res_data_column->get_data());
1187
10
        }
1188
1189
10
        block.get_by_position(result).column =
1190
10
                ColumnNullable::create(std::move(res_data_column), std::move(res_null_map));
1191
10
        return Status::OK();
1192
10
    }
1193
};
1194
1195
class FunctionBitmapToArray : public IFunction {
1196
public:
1197
    static constexpr auto name = "bitmap_to_array";
1198
1199
1
    String get_name() const override { return name; }
1200
1201
14
    static FunctionPtr create() { return std::make_shared<FunctionBitmapToArray>(); }
1202
1203
5
    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
1204
5
        auto nested_type = make_nullable(std::make_shared<DataTypeInt64>());
1205
5
        return std::make_shared<DataTypeArray>(nested_type);
1206
5
    }
1207
1208
5
    size_t get_number_of_arguments() const override { return 1; }
1209
1210
    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
1211
8
                        uint32_t result, size_t input_rows_count) const override {
1212
8
        auto return_nested_type = make_nullable(std::make_shared<DataTypeInt64>());
1213
8
        auto dest_array_column_ptr = ColumnArray::create(return_nested_type->create_column(),
1214
8
                                                         ColumnArray::ColumnOffsets::create());
1215
1216
8
        IColumn* dest_nested_column = &dest_array_column_ptr->get_data();
1217
8
        ColumnNullable* dest_nested_nullable_col =
1218
8
                reinterpret_cast<ColumnNullable*>(dest_nested_column);
1219
8
        dest_nested_column = dest_nested_nullable_col->get_nested_column_ptr().get();
1220
8
        auto& dest_nested_null_map = dest_nested_nullable_col->get_null_map_column().get_data();
1221
1222
8
        auto& arg_col = block.get_by_position(arguments[0]).column;
1223
8
        auto bitmap_col = assert_cast<const ColumnBitmap*>(arg_col.get());
1224
8
        const auto& bitmap_col_data = bitmap_col->get_data();
1225
8
        auto& nested_column_data = assert_cast<ColumnInt64*>(dest_nested_column)->get_data();
1226
8
        auto& dest_offsets = dest_array_column_ptr->get_offsets();
1227
8
        dest_offsets.reserve(input_rows_count);
1228
1229
16
        for (int i = 0; i < input_rows_count; ++i) {
1230
8
            bitmap_col_data[i].to_array(nested_column_data);
1231
8
            dest_nested_null_map.resize_fill(nested_column_data.size(), 0);
1232
8
            dest_offsets.push_back(nested_column_data.size());
1233
8
        }
1234
1235
8
        block.replace_by_position(result, std::move(dest_array_column_ptr));
1236
8
        return Status::OK();
1237
8
    }
1238
};
1239
1240
using FunctionBitmapEmpty = FunctionConst<BitmapEmpty, false>;
1241
using FunctionToBitmap = FunctionAlwaysNotNullable<ToBitmap>;
1242
using FunctionToBitmapWithCheck = FunctionAlwaysNotNullable<ToBitmapWithCheck, true>;
1243
1244
using FunctionBitmapFromString = FunctionBitmapAlwaysNull<BitmapFromString>;
1245
using FunctionBitmapFromArray = FunctionBitmapAlwaysNull<BitmapFromArray>;
1246
using FunctionBitmapHash = FunctionAlwaysNotNullable<BitmapHash<32>>;
1247
using FunctionBitmapHash64 = FunctionAlwaysNotNullable<BitmapHash<64>>;
1248
1249
using FunctionBitmapMin = FunctionBitmapSingle<FunctionBitmapMinImpl>;
1250
using FunctionBitmapMax = FunctionBitmapSingle<FunctionBitmapMaxImpl>;
1251
1252
using FunctionBitmapToString = FunctionUnaryToType<BitmapToString, NameBitmapToString>;
1253
using FunctionBitmapToBase64 = FunctionUnaryToType<BitmapToBase64, NameBitmapToBase64>;
1254
using FunctionBitmapFromBase64 = FunctionBitmapAlwaysNull<BitmapFromBase64>;
1255
using FunctionBitmapNot =
1256
        FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapNot, NameBitmapNot>;
1257
using FunctionBitmapAndNot =
1258
        FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapAndNot, NameBitmapAndNot>;
1259
using FunctionBitmapContains =
1260
        FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapContains, NameBitmapContains>;
1261
using FunctionBitmapRemove =
1262
        FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapRemove, NameBitmapRemove>;
1263
1264
using FunctionBitmapHasAny =
1265
        FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAny, NameBitmapHasAny>;
1266
using FunctionBitmapHasAll =
1267
        FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAll, NameBitmapHasAll>;
1268
using FunctionSubBitmap = FunctionBitmapSubs<SubBitmap>;
1269
using FunctionBitmapSubsetLimit = FunctionBitmapSubs<BitmapSubsetLimit>;
1270
using FunctionBitmapSubsetInRange = FunctionBitmapSubs<BitmapSubsetInRange>;
1271
1272
8
void register_function_bitmap(SimpleFunctionFactory& factory) {
1273
8
    factory.register_function<FunctionBitmapEmpty>();
1274
8
    factory.register_function<FunctionToBitmap>();
1275
8
    factory.register_function<FunctionToBitmapWithCheck>();
1276
8
    factory.register_function<FunctionBitmapFromString>();
1277
8
    factory.register_function<FunctionBitmapToBase64>();
1278
8
    factory.register_function<FunctionBitmapFromBase64>();
1279
8
    factory.register_function<FunctionBitmapFromArray>();
1280
8
    factory.register_function<FunctionBitmapHash>();
1281
8
    factory.register_function<FunctionBitmapHash64>();
1282
8
    factory.register_function<FunctionBitmapCount>();
1283
8
    factory.register_function<FunctionBitmapMin>();
1284
8
    factory.register_function<FunctionBitmapMax>();
1285
8
    factory.register_function<FunctionBitmapToString>();
1286
8
    factory.register_function<FunctionBitmapNot>();
1287
8
    factory.register_function<FunctionBitmapAndNot>();
1288
8
    factory.register_alias(NameBitmapAndNot::name, "bitmap_andnot");
1289
8
    factory.register_function<FunctionBitmapAndNotCount<NameBitmapAndNotCount>>();
1290
8
    factory.register_alias(NameBitmapAndNotCount::name, "bitmap_andnot_count");
1291
8
    factory.register_function<FunctionBitmapContains>();
1292
8
    factory.register_function<FunctionBitmapRemove>();
1293
8
    factory.register_function<FunctionBitmapHasAny>();
1294
8
    factory.register_function<FunctionBitmapHasAll>();
1295
8
    factory.register_function<FunctionSubBitmap>();
1296
8
    factory.register_function<FunctionBitmapSubsetLimit>();
1297
8
    factory.register_function<FunctionBitmapSubsetInRange>();
1298
8
    factory.register_function<FunctionBitmapToArray>();
1299
8
}
1300
1301
} // namespace doris