be/src/util/raw_value.h

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/raw-value.h
// and modified by Doris

#pragma once

#include <string>

#include "common/check.h"
#include "common/consts.h"
#include "common/exception.h"
#include "common/logging.h"
#include "core/data_type/define_primitive_type.h"
#include "core/packed_int128.h"
#include "core/string_ref.h"
#include "util/hash_util.hpp"

namespace doris {
class SlotDescriptor;

// Useful utility functions for runtime values (which are passed around as void*).
class RawValue {
public:
    // Same as the up function, only use in vec exec engine.
    static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type,
                               uint32_t seed);

    // Treat the canonical distribution bytes of a value as an unsigned integer with the first byte
    // as the least-significant byte, then append it to the preceding distribution columns. The
    // returned value is kept modulo mod throughout, so values of any byte width and any number of
    // columns do not require a wide integer.
    static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type,
                                  uint32_t seed, uint32_t mod);

    // The single appending core shared by identity_hash and the batch column implementations in
    // IColumn::update_identity_hash. Interprets size bytes as a little-endian unsigned integer,
    // multiplies the seed by 256^size and adds the value, taking mod every 4 bytes so any byte
    // width never overflows uint64_t. This is the sole source of the identity hash byte semantics
    // and must stay bit-identical with FE HashDistributionPruner and BE tablet routing.
    static uint32_t append_identity_bytes(const void* value, size_t size, uint32_t seed,
                                          uint32_t mod);

    // A null distribution value contributes these four zero canonical bytes, shared by the
    // scalar and batch paths so the null convention has a single definition.
    static constexpr uint32_t NULL_IDENTITY_BYTES = 0;
};

// Keep the canonical byte-width dispatch together so it can be audited against zlib_crc32 below.
// NOLINTNEXTLINE(readability-function-size)
inline uint32_t RawValue::append_identity_bytes(const void* value, size_t size, uint32_t seed,
                                                uint32_t mod) {
    DCHECK_GT(mod, 0);
    const auto* bytes = reinterpret_cast<const uint8_t*>(value);
    uint64_t remainder = seed;
    size_t bytes_since_mod = 0;
    for (size_t i = size; i > 0; --i) {
        remainder = remainder * 256 + bytes[i - 1];
        if (++bytes_since_mod == sizeof(uint32_t)) {
            remainder %= mod;
            bytes_since_mod = 0;
        }
    }
    return static_cast<uint32_t>(remainder % mod);
}

// NOLINTNEXTLINE(readability-function-size)
inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type,
                                        uint32_t seed, uint32_t mod) {
    DCHECK_GT(mod, 0);
    auto append_little_endian = [&seed, mod](const void* value, size_t size) {
        seed = append_identity_bytes(value, size, seed, mod);
    };

    if (v == nullptr) {
        append_little_endian(&NULL_IDENTITY_BYTES, sizeof(NULL_IDENTITY_BYTES));
        return seed;
    }

    switch (type) {
    case TYPE_VARCHAR:
    case TYPE_VARBINARY:
    case TYPE_HLL:
    case TYPE_STRING:
    case TYPE_CHAR:
        append_little_endian(v, len);
        break;
    case TYPE_BOOLEAN:
    case TYPE_TINYINT:
        append_little_endian(v, 1);
        break;
    case TYPE_SMALLINT:
        append_little_endian(v, 2);
        break;
    case TYPE_INT:
    case TYPE_FLOAT:
    case TYPE_DATEV2:
    case TYPE_DECIMAL32:
    case TYPE_IPV4:
        append_little_endian(v, 4);
        break;
    case TYPE_BIGINT:
    case TYPE_DOUBLE:
    case TYPE_TIMEV2:
    case TYPE_DATETIMEV2:
    case TYPE_TIMESTAMP_NS:
    case TYPE_TIMESTAMPTZ:
    case TYPE_DECIMAL64:
        append_little_endian(v, 8);
        break;
    case TYPE_LARGEINT:
    case TYPE_DECIMAL128I:
    case TYPE_IPV6:
    case TYPE_UUID:
        append_little_endian(v, 16);
        break;
    case TYPE_DECIMAL256:
        append_little_endian(v, 32);
        break;
    case TYPE_DATE:
    case TYPE_DATETIME: {
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
        char buf[64];
        int date_len = date_val->to_buffer(buf);
        append_little_endian(buf, date_len);
        break;
    }
    case TYPE_DECIMALV2: {
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
        int64_t int_val = dec_val->int_value();
        int32_t frac_val = dec_val->frac_value();
        append_little_endian(&frac_val, sizeof(frac_val));
        append_little_endian(&int_val, sizeof(int_val));
        break;
    }
    default:
        DORIS_CHECK(false) << "invalid type: " << type;
    }
    return seed;
}

// NOTE: this is just for split data, decimal use old doris hash function
// Because crc32 hardware is not equal with zlib crc32
inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type,
                                     uint32_t seed) {
    // Reject binary even for NULL instead of reaching the default-type assertion or hash path.
    if (type == TYPE_VARBINARY) {
        throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
                        "VARBINARY tablet routing hash is not supported");
    }
    // Hash_combine with v = 0
    if (v == nullptr) {
        uint32_t value = 0x9e3779b9;
        return seed ^ (value + (seed << 6) + (seed >> 2));
    }

    switch (type) {
    case TYPE_VARCHAR:
    case TYPE_HLL:
    case TYPE_STRING:
    case TYPE_CHAR: {
        return HashUtil::zlib_crc_hash(v, (uint32_t)len, seed);
    }

    case TYPE_BOOLEAN:
    case TYPE_TINYINT:
        return HashUtil::zlib_crc_hash(v, 1, seed);
    case TYPE_SMALLINT:
        return HashUtil::zlib_crc_hash(v, 2, seed);
    case TYPE_INT:
        return HashUtil::zlib_crc_hash(v, 4, seed);
    case TYPE_BIGINT:
        return HashUtil::zlib_crc_hash(v, 8, seed);
    case TYPE_LARGEINT:
        return HashUtil::zlib_crc_hash(v, 16, seed);
    case TYPE_FLOAT:
        return HashUtil::zlib_crc_hash(v, 4, seed);
    case TYPE_DOUBLE:
        return HashUtil::zlib_crc_hash(v, 8, seed);
    case TYPE_DATE:
    case TYPE_DATETIME: {
        const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
        char buf[64];
        int date_len = date_val->to_buffer(buf);
        return HashUtil::zlib_crc_hash(buf, date_len, seed);
    }

    case TYPE_DATEV2: {
        return HashUtil::zlib_crc_hash(v, 4, seed);
    }

    case TYPE_DATETIMEV2:
    case TYPE_TIMESTAMP_NS: {
        return HashUtil::zlib_crc_hash(v, 8, seed);
    }

    case TYPE_TIMESTAMPTZ: {
        return HashUtil::zlib_crc_hash(v, 8, seed);
    }

    case TYPE_DECIMALV2: {
        const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
        int64_t int_val = dec_val->int_value();
        int32_t frac_val = dec_val->frac_value();
        seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
        return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
    }
    case TYPE_DECIMAL32:
        return HashUtil::zlib_crc_hash(v, 4, seed);
    case TYPE_DECIMAL64:
        return HashUtil::zlib_crc_hash(v, 8, seed);
    case TYPE_DECIMAL128I:
        return HashUtil::zlib_crc_hash(v, 16, seed);
    case TYPE_DECIMAL256:
        return HashUtil::zlib_crc_hash(v, 32, seed);
    case TYPE_IPV4:
        return HashUtil::zlib_crc_hash(v, 4, seed);
    case TYPE_IPV6:
    case TYPE_UUID:
        return HashUtil::zlib_crc_hash(v, 16, seed);
    default:
        DCHECK(false) << "invalid type: " << type;
        return 0;
    }
}
} // namespace doris

Coverage Report

Created: 2026-11-02 21:18

Line	Count	Source
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17		// This file is copied from
18		// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/raw-value.h
19		// and modified by Doris
20
21		#pragma once
22
23		#include <string>
24
25		#include "common/check.h"
26		#include "common/consts.h"
27		#include "common/exception.h"
28		#include "common/logging.h"
29		#include "core/data_type/define_primitive_type.h"
30		#include "core/packed_int128.h"
31		#include "core/string_ref.h"
32		#include "util/hash_util.hpp"
33
34		namespace doris {
35		class SlotDescriptor;
36
37		// Useful utility functions for runtime values (which are passed around as void*).
38		class RawValue {
39		public:
40		// Same as the up function, only use in vec exec engine.
41		static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type,
42		uint32_t seed);
43
44		// Treat the canonical distribution bytes of a value as an unsigned integer with the first byte
45		// as the least-significant byte, then append it to the preceding distribution columns. The
46		// returned value is kept modulo mod throughout, so values of any byte width and any number of
47		// columns do not require a wide integer.
48		static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type,
49		uint32_t seed, uint32_t mod);
50
51		// The single appending core shared by identity_hash and the batch column implementations in
52		// IColumn::update_identity_hash. Interprets size bytes as a little-endian unsigned integer,
53		// multiplies the seed by 256^size and adds the value, taking mod every 4 bytes so any byte
54		// width never overflows uint64_t. This is the sole source of the identity hash byte semantics
55		// and must stay bit-identical with FE HashDistributionPruner and BE tablet routing.
56		static uint32_t append_identity_bytes(const void* value, size_t size, uint32_t seed,
57		uint32_t mod);
58
59		// A null distribution value contributes these four zero canonical bytes, shared by the
60		// scalar and batch paths so the null convention has a single definition.
61		static constexpr uint32_t NULL_IDENTITY_BYTES = 0;
62		};
63
64		// Keep the canonical byte-width dispatch together so it can be audited against zlib_crc32 below.
65		// NOLINTNEXTLINE(readability-function-size)
66		inline uint32_t RawValue::append_identity_bytes(const void* value, size_t size, uint32_t seed,
67	1.19M	uint32_t mod) {
68	1.19M	DCHECK_GT(mod, 0);
69	1.19M	const auto* bytes = reinterpret_cast<const uint8_t*>(value);
70	1.19M	uint64_t remainder = seed;
71	1.19M	size_t bytes_since_mod = 0;
72	6.03M	for (size_t i = size; i > 0; --i) {
73	4.83M	remainder = remainder * 256 + bytes[i - 1];
74	4.83M	if (++bytes_since_mod == sizeof(uint32_t)) {
75	1.20M	remainder %= mod;
76	1.20M	bytes_since_mod = 0;
77	1.20M	}
78	4.83M	}
79	1.19M	return static_cast<uint32_t>(remainder % mod);
80	1.19M	}
81
82		// NOLINTNEXTLINE(readability-function-size)
83		inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type,
84	1.19M	uint32_t seed, uint32_t mod) {
85	1.19M	DCHECK_GT(mod, 0);
86	1.19M	auto append_little_endian = [&seed, mod](const void* value, size_t size) {
87	1.19M	seed = append_identity_bytes(value, size, seed, mod);
88	1.19M	};
89
90	1.19M	if (v == nullptr) {
91	837	append_little_endian(&NULL_IDENTITY_BYTES, sizeof(NULL_IDENTITY_BYTES));
92	837	return seed;
93	837	}
94
95	1.19M	switch (type) {
96	25	case TYPE_VARCHAR:
97	25	case TYPE_VARBINARY:
98	25	case TYPE_HLL:
99	52	case TYPE_STRING:
100	56	case TYPE_CHAR:
101	56	append_little_endian(v, len);
102	56	break;
103	22	case TYPE_BOOLEAN:
104	44	case TYPE_TINYINT:
105	44	append_little_endian(v, 1);
106	44	break;
107	22	case TYPE_SMALLINT:
108	22	append_little_endian(v, 2);
109	22	break;
110	1.18M	case TYPE_INT:
111	1.18M	case TYPE_FLOAT:
112	1.18M	case TYPE_DATEV2:
113	1.18M	case TYPE_DECIMAL32:
114	1.18M	case TYPE_IPV4:
115	1.18M	append_little_endian(v, 4);
116	1.18M	break;
117	8.55k	case TYPE_BIGINT:
118	8.56k	case TYPE_DOUBLE:
119	8.58k	case TYPE_TIMEV2:
120	8.61k	case TYPE_DATETIMEV2:
121	8.64k	case TYPE_TIMESTAMP_NS:
122	8.66k	case TYPE_TIMESTAMPTZ:
123	8.70k	case TYPE_DECIMAL64:
124	8.70k	append_little_endian(v, 8);
125	8.70k	break;
126	31	case TYPE_LARGEINT:
127	49	case TYPE_DECIMAL128I:
128	72	case TYPE_IPV6:
129	82	case TYPE_UUID:
130	82	append_little_endian(v, 16);
131	82	break;
132	28	case TYPE_DECIMAL256:
133	28	append_little_endian(v, 32);
134	28	break;
135	30	case TYPE_DATE:
136	39	case TYPE_DATETIME: {
137	39	const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
138	39	char buf[64];
139	39	int date_len = date_val->to_buffer(buf);
140	39	append_little_endian(buf, date_len);
141	39	break;
142	30	}
143	27	case TYPE_DECIMALV2: {
144	27	const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
145	27	int64_t int_val = dec_val->int_value();
146	27	int32_t frac_val = dec_val->frac_value();
147	27	append_little_endian(&frac_val, sizeof(frac_val));
148	27	append_little_endian(&int_val, sizeof(int_val));
149	27	break;
150	30	}
151	0	default:
152	0	DORIS_CHECK(false) << "invalid type: " << type;
153	1.19M	}
154	1.19M	return seed;
155	1.19M	}
156
157		// NOTE: this is just for split data, decimal use old doris hash function
158		// Because crc32 hardware is not equal with zlib crc32
159		inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type,
160	40.9M	uint32_t seed) {
161		// Reject binary even for NULL instead of reaching the default-type assertion or hash path.
162	40.9M	if (type == TYPE_VARBINARY) {
163	3	throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
164	3	"VARBINARY tablet routing hash is not supported");
165	3	}
166		// Hash_combine with v = 0
167	40.9M	if (v == nullptr) {
168	0	uint32_t value = 0x9e3779b9;
169	0	return seed ^ (value + (seed << 6) + (seed >> 2));
170	0	}
171
172	40.9M	switch (type) {
173	968k	case TYPE_VARCHAR:
174	968k	case TYPE_HLL:
175	968k	case TYPE_STRING:
176	969k	case TYPE_CHAR: {
177	969k	return HashUtil::zlib_crc_hash(v, (uint32_t)len, seed);
178	968k	}
179
180	135	case TYPE_BOOLEAN:
181	3.11k	case TYPE_TINYINT:
182	3.11k	return HashUtil::zlib_crc_hash(v, 1, seed);
183	1.31k	case TYPE_SMALLINT:
184	1.31k	return HashUtil::zlib_crc_hash(v, 2, seed);
185	22.9M	case TYPE_INT:
186	22.9M	return HashUtil::zlib_crc_hash(v, 4, seed);
187	16.9M	case TYPE_BIGINT:
188	16.9M	return HashUtil::zlib_crc_hash(v, 8, seed);
189	28.1k	case TYPE_LARGEINT:
190	28.1k	return HashUtil::zlib_crc_hash(v, 16, seed);
191	0	case TYPE_FLOAT:
192	0	return HashUtil::zlib_crc_hash(v, 4, seed);
193	0	case TYPE_DOUBLE:
194	0	return HashUtil::zlib_crc_hash(v, 8, seed);
195	25	case TYPE_DATE:
196	50	case TYPE_DATETIME: {
197	50	const auto* date_val = reinterpret_cast<const VecDateTimeValue*>(v);
198	50	char buf[64];
199	50	int date_len = date_val->to_buffer(buf);
200	50	return HashUtil::zlib_crc_hash(buf, date_len, seed);
201	25	}
202
203	1.77k	case TYPE_DATEV2: {
204	1.77k	return HashUtil::zlib_crc_hash(v, 4, seed);
205	25	}
206
207	12.4k	case TYPE_DATETIMEV2:
208	12.5k	case TYPE_TIMESTAMP_NS: {
209	12.5k	return HashUtil::zlib_crc_hash(v, 8, seed);
210	12.4k	}
211
212	2.93k	case TYPE_TIMESTAMPTZ: {
213	2.93k	return HashUtil::zlib_crc_hash(v, 8, seed);
214	12.4k	}
215
216	27	case TYPE_DECIMALV2: {
217	27	const auto* dec_val = reinterpret_cast<const DecimalV2Value*>(v);
218	27	int64_t int_val = dec_val->int_value();
219	27	int32_t frac_val = dec_val->frac_value();
220	27	seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed);
221	27	return HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), seed);
222	12.4k	}
223	156	case TYPE_DECIMAL32:
224	156	return HashUtil::zlib_crc_hash(v, 4, seed);
225	224	case TYPE_DECIMAL64:
226	224	return HashUtil::zlib_crc_hash(v, 8, seed);
227	428	case TYPE_DECIMAL128I:
228	428	return HashUtil::zlib_crc_hash(v, 16, seed);
229	4.37k	case TYPE_DECIMAL256:
230	4.37k	return HashUtil::zlib_crc_hash(v, 32, seed);
231	3	case TYPE_IPV4:
232	3	return HashUtil::zlib_crc_hash(v, 4, seed);
233	3	case TYPE_IPV6:
234	125	case TYPE_UUID:
235	125	return HashUtil::zlib_crc_hash(v, 16, seed);
236	0	default:
237	0	DCHECK(false) << "invalid type: " << type;
238	0	return 0;
239	40.9M	}
240	40.9M	}
241		} // namespace doris