Coverage Report

Created: 2026-09-12 03:45

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/raw_value.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
// This file is copied from
18
// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/raw-value.h
19
// and modified by Doris
20
21
#pragma once
22
23
#include <string>
24
25
#include "common/check.h"
26
#include "common/consts.h"
27
#include "common/logging.h"
28
#include "core/data_type/define_primitive_type.h"
29
#include "core/packed_int128.h"
30
#include "core/string_ref.h"
31
#include "util/hash_util.hpp"
32
33
namespace doris {
34
class SlotDescriptor;
35
36
// Useful utility functions for runtime values (which are passed around as void*).
37
class RawValue {
38
public:
39
    // Same as the up function, only use in vec exec engine.
40
    static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type,
41
                               uint32_t seed);
42
43
    // Treat the canonical distribution bytes of a value as an unsigned integer with the first byte
44
    // as the least-significant byte, then append it to the preceding distribution columns. The
45
    // returned value is kept modulo mod throughout, so values of any byte width and any number of
46
    // columns do not require a wide integer.
47
    static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type,
48
                                  uint32_t seed, uint32_t mod);
49
};
50
51
inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type,
52
47
                                        uint32_t seed, uint32_t mod) {
53
47
    DCHECK_GT(mod, 0);
54
48
    auto append_little_endian = [&seed, mod](const void* value, size_t size) {
55
48
        const auto* bytes = reinterpret_cast<const uint8_t*>(value);
56
48
        uint64_t remainder = seed;
57
48
        size_t bytes_since_mod = 0;
58
294
        for (size_t i = size; i > 0; --i) {
59
246
            remainder = remainder * 256 + bytes[i - 1];
60
246
            if (++bytes_since_mod == sizeof(uint32_t)) {
61
59
                remainder %= mod;
62
59
                bytes_since_mod = 0;
63
59
            }
64
246
        }
65
48
        seed = static_cast<uint32_t>(remainder % mod);
66
48
    };
67
68
47
    if (v == nullptr) {
69
1
        static constexpr uint32_t NULL_VALUE = 0;
70
1
        append_little_endian(&NULL_VALUE, sizeof(NULL_VALUE));
71
1
        return seed;
72
1
    }
73
74
46
    switch (type) {
75
1
    case TYPE_VARCHAR:
76
1
    case TYPE_VARBINARY:
77
1
    case TYPE_HLL:
78
3
    case TYPE_STRING:
79
3
    case TYPE_CHAR:
80
3
        append_little_endian(v, len);
81
3
        break;
82
1
    case TYPE_BOOLEAN:
83
1
    case TYPE_TINYINT:
84
1
        append_little_endian(v, 1);
85
1
        break;
86
1
    case TYPE_SMALLINT:
87
1
        append_little_endian(v, 2);
88
1
        break;
89
34
    case TYPE_INT:
90
34
    case TYPE_FLOAT:
91
34
    case TYPE_DATEV2:
92
34
    case TYPE_DECIMAL32:
93
35
    case TYPE_IPV4:
94
35
        append_little_endian(v, 4);
95
35
        break;
96
1
    case TYPE_BIGINT:
97
1
    case TYPE_DOUBLE:
98
1
    case TYPE_TIMEV2:
99
1
    case TYPE_DATETIMEV2:
100
1
    case TYPE_TIMESTAMPTZ:
101
1
    case TYPE_DECIMAL64:
102
1
        append_little_endian(v, 8);
103
1
        break;
104
1
    case TYPE_LARGEINT:
105
1
    case TYPE_DECIMAL128I:
106
2
    case TYPE_IPV6:
107
2
        append_little_endian(v, 16);
108
2
        break;
109
1
    case TYPE_DECIMAL256:
110
1
        append_little_endian(v, 32);
111
1
        break;
112
1
    case TYPE_DATE:
113
1
    case TYPE_DATETIME: {
114
1
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
115
1
        char buf[64];
116
1
        int date_len = date_val->to_buffer(buf);
117
1
        append_little_endian(buf, date_len);
118
1
        break;
119
1
    }
120
1
    case TYPE_DECIMALV2: {
121
1
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
122
1
        int64_t int_val = dec_val->int_value();
123
1
        int32_t frac_val = dec_val->frac_value();
124
1
        append_little_endian(&frac_val, sizeof(frac_val));
125
1
        append_little_endian(&int_val, sizeof(int_val));
126
1
        break;
127
1
    }
128
0
    default:
129
0
        DORIS_CHECK(false) << "invalid type: " << type;
130
46
    }
131
46
    return seed;
132
46
}
133
134
// NOTE: this is just for split data, decimal use old doris hash function
135
// Because crc32 hardware is not equal with zlib crc32
136
inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type,
137
40
                                     uint32_t seed) {
138
    // Hash_combine with v = 0
139
40
    if (v == nullptr) {
140
0
        uint32_t value = 0x9e3779b9;
141
0
        return seed ^ (value + (seed << 6) + (seed >> 2));
142
0
    }
143
144
40
    switch (type) {
145
0
    case TYPE_VARCHAR:
146
0
    case TYPE_HLL:
147
0
    case TYPE_STRING:
148
0
    case TYPE_CHAR: {
149
0
        return HashUtil::zlib_crc_hash(v, (uint32_t)len, seed);
150
0
    }
151
152
0
    case TYPE_BOOLEAN:
153
0
    case TYPE_TINYINT:
154
0
        return HashUtil::zlib_crc_hash(v, 1, seed);
155
0
    case TYPE_SMALLINT:
156
0
        return HashUtil::zlib_crc_hash(v, 2, seed);
157
30
    case TYPE_INT:
158
30
        return HashUtil::zlib_crc_hash(v, 4, seed);
159
0
    case TYPE_BIGINT:
160
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
161
0
    case TYPE_LARGEINT:
162
0
        return HashUtil::zlib_crc_hash(v, 16, seed);
163
0
    case TYPE_FLOAT:
164
0
        return HashUtil::zlib_crc_hash(v, 4, seed);
165
0
    case TYPE_DOUBLE:
166
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
167
0
    case TYPE_DATE:
168
0
    case TYPE_DATETIME: {
169
0
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
170
0
        char buf[64];
171
0
        int date_len = date_val->to_buffer(buf);
172
0
        return HashUtil::zlib_crc_hash(buf, date_len, seed);
173
0
    }
174
175
0
    case TYPE_DATEV2: {
176
0
        return HashUtil::zlib_crc_hash(v, 4, seed);
177
0
    }
178
179
0
    case TYPE_DATETIMEV2:
180
0
    case TYPE_TIMESTAMP_NS: {
181
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
182
0
    }
183
184
0
    case TYPE_TIMESTAMPTZ: {
185
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
186
0
    }
187
188
0
    case TYPE_DECIMALV2: {
189
0
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
190
0
        int64_t int_val = dec_val->int_value();
191
0
        int32_t frac_val = dec_val->frac_value();
192
0
        seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
193
0
        return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
194
0
    }
195
0
    case TYPE_DECIMAL32:
196
0
        return HashUtil::zlib_crc_hash(v, 4, seed);
197
0
    case TYPE_DECIMAL64:
198
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
199
0
    case TYPE_DECIMAL128I:
200
0
        return HashUtil::zlib_crc_hash(v, 16, seed);
201
4
    case TYPE_DECIMAL256:
202
4
        return HashUtil::zlib_crc_hash(v, 32, seed);
203
3
    case TYPE_IPV4:
204
3
        return HashUtil::zlib_crc_hash(v, 4, seed);
205
3
    case TYPE_IPV6:
206
3
        return HashUtil::zlib_crc_hash(v, 16, seed);
207
0
    default:
208
0
        DCHECK(false) << "invalid type: " << type;
209
0
        return 0;
210
40
    }
211
40
}
212
} // namespace doris