Coverage Report

Created: 2026-09-02 17:47

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/raw_value.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
// This file is copied from
18
// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/raw-value.h
19
// and modified by Doris
20
21
#pragma once
22
23
#include <string>
24
25
#include "common/check.h"
26
#include "common/consts.h"
27
#include "common/logging.h"
28
#include "core/data_type/define_primitive_type.h"
29
#include "core/packed_int128.h"
30
#include "core/string_ref.h"
31
#include "util/hash_util.hpp"
32
33
namespace doris {
34
class SlotDescriptor;
35
36
// Useful utility functions for runtime values (which are passed around as void*).
37
class RawValue {
38
public:
39
    // Same as the up function, only use in vec exec engine.
40
    static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type,
41
                               uint32_t seed);
42
43
    // Treat the canonical distribution bytes of a value as an unsigned integer with the first byte
44
    // as the least-significant byte, then append it to the preceding distribution columns. The
45
    // returned value is kept modulo mod throughout, so values of any byte width and any number of
46
    // columns do not require a wide integer.
47
    static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type,
48
                                  uint32_t seed, uint32_t mod);
49
};
50
51
inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type,
52
242
                                        uint32_t seed, uint32_t mod) {
53
242
    DCHECK_GT(mod, 0);
54
242
    auto append_little_endian = [&seed, mod](const void* value, size_t size) {
55
242
        const auto* bytes = reinterpret_cast<const uint8_t*>(value);
56
242
        uint64_t remainder = seed;
57
242
        size_t bytes_since_mod = 0;
58
1.74k
        for (size_t i = size; i > 0; --i) {
59
1.50k
            remainder = remainder * 256 + bytes[i - 1];
60
1.50k
            if (++bytes_since_mod == sizeof(uint32_t)) {
61
364
                remainder %= mod;
62
364
                bytes_since_mod = 0;
63
364
            }
64
1.50k
        }
65
242
        seed = static_cast<uint32_t>(remainder % mod);
66
242
    };
67
68
242
    if (v == nullptr) {
69
5
        static constexpr uint32_t NULL_VALUE = 0;
70
5
        append_little_endian(&NULL_VALUE, sizeof(NULL_VALUE));
71
5
        return seed;
72
5
    }
73
74
237
    switch (type) {
75
30
    case TYPE_VARCHAR:
76
30
    case TYPE_VARBINARY:
77
30
    case TYPE_HLL:
78
32
    case TYPE_STRING:
79
32
    case TYPE_CHAR:
80
32
        append_little_endian(v, len);
81
32
        break;
82
0
    case TYPE_BOOLEAN:
83
0
    case TYPE_TINYINT:
84
0
        append_little_endian(v, 1);
85
0
        break;
86
0
    case TYPE_SMALLINT:
87
0
        append_little_endian(v, 2);
88
0
        break;
89
58
    case TYPE_INT:
90
58
    case TYPE_FLOAT:
91
59
    case TYPE_DATEV2:
92
59
    case TYPE_DECIMAL32:
93
62
    case TYPE_IPV4:
94
62
        append_little_endian(v, 4);
95
62
        break;
96
138
    case TYPE_BIGINT:
97
138
    case TYPE_DOUBLE:
98
138
    case TYPE_TIMEV2:
99
139
    case TYPE_DATETIMEV2:
100
139
    case TYPE_TIMESTAMPTZ:
101
140
    case TYPE_DECIMAL64:
102
140
        append_little_endian(v, 8);
103
140
        break;
104
0
    case TYPE_LARGEINT:
105
0
    case TYPE_DECIMAL128I:
106
3
    case TYPE_IPV6:
107
3
        append_little_endian(v, 16);
108
3
        break;
109
0
    case TYPE_DECIMAL256:
110
0
        append_little_endian(v, 32);
111
0
        break;
112
0
    case TYPE_DATE:
113
0
    case TYPE_DATETIME: {
114
0
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
115
0
        char buf[64];
116
0
        int date_len = date_val->to_buffer(buf);
117
0
        append_little_endian(buf, date_len);
118
0
        break;
119
0
    }
120
0
    case TYPE_DECIMALV2: {
121
0
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
122
0
        int64_t int_val = dec_val->int_value();
123
0
        int32_t frac_val = dec_val->frac_value();
124
0
        append_little_endian(&frac_val, sizeof(frac_val));
125
0
        append_little_endian(&int_val, sizeof(int_val));
126
0
        break;
127
0
    }
128
0
    default:
129
0
        DORIS_CHECK(false) << "invalid type: " << type;
130
237
    }
131
237
    return seed;
132
237
}
133
134
// NOTE: this is just for split data, decimal use old doris hash function
135
// Because crc32 hardware is not equal with zlib crc32
136
inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type,
137
30.8M
                                     uint32_t seed) {
138
    // Hash_combine with v = 0
139
30.8M
    if (v == nullptr) {
140
0
        uint32_t value = 0x9e3779b9;
141
0
        return seed ^ (value + (seed << 6) + (seed >> 2));
142
0
    }
143
144
30.8M
    switch (type) {
145
815k
    case TYPE_VARCHAR:
146
815k
    case TYPE_HLL:
147
815k
    case TYPE_STRING:
148
816k
    case TYPE_CHAR: {
149
816k
        return HashUtil::zlib_crc_hash(v, (uint32_t)len, seed);
150
815k
    }
151
152
130
    case TYPE_BOOLEAN:
153
2.56k
    case TYPE_TINYINT:
154
2.56k
        return HashUtil::zlib_crc_hash(v, 1, seed);
155
1.29k
    case TYPE_SMALLINT:
156
1.29k
        return HashUtil::zlib_crc_hash(v, 2, seed);
157
19.0M
    case TYPE_INT:
158
19.0M
        return HashUtil::zlib_crc_hash(v, 4, seed);
159
10.9M
    case TYPE_BIGINT:
160
10.9M
        return HashUtil::zlib_crc_hash(v, 8, seed);
161
27.4k
    case TYPE_LARGEINT:
162
27.4k
        return HashUtil::zlib_crc_hash(v, 16, seed);
163
0
    case TYPE_FLOAT:
164
0
        return HashUtil::zlib_crc_hash(v, 4, seed);
165
0
    case TYPE_DOUBLE:
166
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
167
22
    case TYPE_DATE:
168
47
    case TYPE_DATETIME: {
169
47
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
170
47
        char buf[64];
171
47
        int date_len = date_val->to_buffer(buf);
172
47
        return HashUtil::zlib_crc_hash(buf, date_len, seed);
173
22
    }
174
175
1.31k
    case TYPE_DATEV2: {
176
1.31k
        return HashUtil::zlib_crc_hash(v, 4, seed);
177
22
    }
178
179
7.20k
    case TYPE_DATETIMEV2: {
180
7.20k
        return HashUtil::zlib_crc_hash(v, 8, seed);
181
22
    }
182
183
2.93k
    case TYPE_TIMESTAMPTZ: {
184
2.93k
        return HashUtil::zlib_crc_hash(v, 8, seed);
185
22
    }
186
187
27
    case TYPE_DECIMALV2: {
188
27
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
189
27
        int64_t int_val = dec_val->int_value();
190
27
        int32_t frac_val = dec_val->frac_value();
191
27
        seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
192
27
        return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
193
22
    }
194
135
    case TYPE_DECIMAL32:
195
135
        return HashUtil::zlib_crc_hash(v, 4, seed);
196
215
    case TYPE_DECIMAL64:
197
215
        return HashUtil::zlib_crc_hash(v, 8, seed);
198
383
    case TYPE_DECIMAL128I:
199
383
        return HashUtil::zlib_crc_hash(v, 16, seed);
200
4.37k
    case TYPE_DECIMAL256:
201
4.37k
        return HashUtil::zlib_crc_hash(v, 32, seed);
202
5
    case TYPE_IPV4:
203
5
        return HashUtil::zlib_crc_hash(v, 4, seed);
204
5
    case TYPE_IPV6:
205
5
        return HashUtil::zlib_crc_hash(v, 16, seed);
206
0
    default:
207
0
        DCHECK(false) << "invalid type: " << type;
208
0
        return 0;
209
30.8M
    }
210
30.8M
}
211
} // namespace doris