Coverage Report

Created: 2026-09-24 10:09

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/raw_value.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
// This file is copied from
18
// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/raw-value.h
19
// and modified by Doris
20
21
#pragma once
22
23
#include <string>
24
25
#include "common/check.h"
26
#include "common/consts.h"
27
#include "common/exception.h"
28
#include "common/logging.h"
29
#include "core/data_type/define_primitive_type.h"
30
#include "core/packed_int128.h"
31
#include "core/string_ref.h"
32
#include "util/hash_util.hpp"
33
34
namespace doris {
35
class SlotDescriptor;
36
37
// Useful utility functions for runtime values (which are passed around as void*).
38
class RawValue {
39
public:
40
    // Same as the up function, only use in vec exec engine.
41
    static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type,
42
                               uint32_t seed);
43
44
    // Treat the canonical distribution bytes of a value as an unsigned integer with the first byte
45
    // as the least-significant byte, then append it to the preceding distribution columns. The
46
    // returned value is kept modulo mod throughout, so values of any byte width and any number of
47
    // columns do not require a wide integer.
48
    static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type,
49
                                  uint32_t seed, uint32_t mod);
50
};
51
52
// Keep the canonical byte-width dispatch together so it can be audited against zlib_crc32 below.
53
// NOLINTNEXTLINE(readability-function-size)
54
inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type,
55
1.19M
                                        uint32_t seed, uint32_t mod) {
56
1.19M
    DCHECK_GT(mod, 0);
57
1.19M
    auto append_little_endian = [&seed, mod](const void* value, size_t size) {
58
1.19M
        const auto* bytes = reinterpret_cast<const uint8_t*>(value);
59
1.19M
        uint64_t remainder = seed;
60
1.19M
        size_t bytes_since_mod = 0;
61
6.03M
        for (size_t i = size; i > 0; --i) {
62
4.83M
            remainder = remainder * 256 + bytes[i - 1];
63
4.83M
            if (++bytes_since_mod == sizeof(uint32_t)) {
64
1.20M
                remainder %= mod;
65
1.20M
                bytes_since_mod = 0;
66
1.20M
            }
67
4.83M
        }
68
1.19M
        seed = static_cast<uint32_t>(remainder % mod);
69
1.19M
    };
70
71
1.19M
    if (v == nullptr) {
72
802
        static constexpr uint32_t NULL_VALUE = 0;
73
802
        append_little_endian(&NULL_VALUE, sizeof(NULL_VALUE));
74
802
        return seed;
75
802
    }
76
77
1.19M
    switch (type) {
78
29
    case TYPE_VARCHAR:
79
29
    case TYPE_VARBINARY:
80
29
    case TYPE_HLL:
81
37
    case TYPE_STRING:
82
41
    case TYPE_CHAR:
83
41
        append_little_endian(v, len);
84
41
        break;
85
22
    case TYPE_BOOLEAN:
86
44
    case TYPE_TINYINT:
87
44
        append_little_endian(v, 1);
88
44
        break;
89
22
    case TYPE_SMALLINT:
90
22
        append_little_endian(v, 2);
91
22
        break;
92
1.18M
    case TYPE_INT:
93
1.18M
    case TYPE_FLOAT:
94
1.18M
    case TYPE_DATEV2:
95
1.18M
    case TYPE_DECIMAL32:
96
1.18M
    case TYPE_IPV4:
97
1.18M
        append_little_endian(v, 4);
98
1.18M
        break;
99
8.61k
    case TYPE_BIGINT:
100
8.63k
    case TYPE_DOUBLE:
101
8.65k
    case TYPE_TIMEV2:
102
8.67k
    case TYPE_DATETIMEV2:
103
8.69k
    case TYPE_TIMESTAMP_NS:
104
8.71k
    case TYPE_TIMESTAMPTZ:
105
8.74k
    case TYPE_DECIMAL64:
106
8.74k
        append_little_endian(v, 8);
107
8.74k
        break;
108
22
    case TYPE_LARGEINT:
109
40
    case TYPE_DECIMAL128I:
110
63
    case TYPE_IPV6:
111
63
        append_little_endian(v, 16);
112
63
        break;
113
28
    case TYPE_DECIMAL256:
114
28
        append_little_endian(v, 32);
115
28
        break;
116
15
    case TYPE_DATE:
117
15
    case TYPE_DATETIME: {
118
15
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
119
15
        char buf[64];
120
15
        int date_len = date_val->to_buffer(buf);
121
15
        append_little_endian(buf, date_len);
122
15
        break;
123
15
    }
124
18
    case TYPE_DECIMALV2: {
125
18
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
126
18
        int64_t int_val = dec_val->int_value();
127
18
        int32_t frac_val = dec_val->frac_value();
128
18
        append_little_endian(&frac_val, sizeof(frac_val));
129
18
        append_little_endian(&int_val, sizeof(int_val));
130
18
        break;
131
15
    }
132
0
    default:
133
0
        DORIS_CHECK(false) << "invalid type: " << type;
134
1.19M
    }
135
1.19M
    return seed;
136
1.19M
}
137
138
// NOTE: this is just for split data, decimal use old doris hash function
139
// Because crc32 hardware is not equal with zlib crc32
140
inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type,
141
39.5M
                                     uint32_t seed) {
142
    // Reject binary even for NULL instead of reaching the default-type assertion or hash path.
143
39.5M
    if (type == TYPE_VARBINARY) {
144
3
        throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
145
3
                        "VARBINARY tablet routing hash is not supported");
146
3
    }
147
    // Hash_combine with v = 0
148
39.5M
    if (v == nullptr) {
149
0
        uint32_t value = 0x9e3779b9;
150
0
        return seed ^ (value + (seed << 6) + (seed >> 2));
151
0
    }
152
153
39.5M
    switch (type) {
154
956k
    case TYPE_VARCHAR:
155
956k
    case TYPE_HLL:
156
956k
    case TYPE_STRING:
157
957k
    case TYPE_CHAR: {
158
957k
        return HashUtil::zlib_crc_hash(v, (uint32_t)len, seed);
159
956k
    }
160
161
135
    case TYPE_BOOLEAN:
162
3.09k
    case TYPE_TINYINT:
163
3.09k
        return HashUtil::zlib_crc_hash(v, 1, seed);
164
1.31k
    case TYPE_SMALLINT:
165
1.31k
        return HashUtil::zlib_crc_hash(v, 2, seed);
166
21.7M
    case TYPE_INT:
167
21.7M
        return HashUtil::zlib_crc_hash(v, 4, seed);
168
16.7M
    case TYPE_BIGINT:
169
16.7M
        return HashUtil::zlib_crc_hash(v, 8, seed);
170
28.3k
    case TYPE_LARGEINT:
171
28.3k
        return HashUtil::zlib_crc_hash(v, 16, seed);
172
0
    case TYPE_FLOAT:
173
0
        return HashUtil::zlib_crc_hash(v, 4, seed);
174
0
    case TYPE_DOUBLE:
175
0
        return HashUtil::zlib_crc_hash(v, 8, seed);
176
25
    case TYPE_DATE:
177
50
    case TYPE_DATETIME: {
178
50
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
179
50
        char buf[64];
180
50
        int date_len = date_val->to_buffer(buf);
181
50
        return HashUtil::zlib_crc_hash(buf, date_len, seed);
182
25
    }
183
184
1.76k
    case TYPE_DATEV2: {
185
1.76k
        return HashUtil::zlib_crc_hash(v, 4, seed);
186
25
    }
187
188
12.4k
    case TYPE_DATETIMEV2:
189
12.5k
    case TYPE_TIMESTAMP_NS: {
190
12.5k
        return HashUtil::zlib_crc_hash(v, 8, seed);
191
12.4k
    }
192
193
2.93k
    case TYPE_TIMESTAMPTZ: {
194
2.93k
        return HashUtil::zlib_crc_hash(v, 8, seed);
195
12.4k
    }
196
197
27
    case TYPE_DECIMALV2: {
198
27
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
199
27
        int64_t int_val = dec_val->int_value();
200
27
        int32_t frac_val = dec_val->frac_value();
201
27
        seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
202
27
        return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
203
12.4k
    }
204
156
    case TYPE_DECIMAL32:
205
156
        return HashUtil::zlib_crc_hash(v, 4, seed);
206
224
    case TYPE_DECIMAL64:
207
224
        return HashUtil::zlib_crc_hash(v, 8, seed);
208
428
    case TYPE_DECIMAL128I:
209
428
        return HashUtil::zlib_crc_hash(v, 16, seed);
210
4.37k
    case TYPE_DECIMAL256:
211
4.37k
        return HashUtil::zlib_crc_hash(v, 32, seed);
212
3
    case TYPE_IPV4:
213
3
        return HashUtil::zlib_crc_hash(v, 4, seed);
214
3
    case TYPE_IPV6:
215
3
        return HashUtil::zlib_crc_hash(v, 16, seed);
216
0
    default:
217
0
        DCHECK(false) << "invalid type: " << type;
218
0
        return 0;
219
39.5M
    }
220
39.5M
}
221
} // namespace doris