Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <array> |
21 | | #include <cstddef> |
22 | | #include <cstdint> |
23 | | #include <memory> |
24 | | #include <string> |
25 | | |
26 | | #include "util/variant/variant_value.h" |
27 | | |
28 | | namespace doris { |
29 | | |
30 | | class SipHash; |
31 | | class CanonicalScalarSerializationPlan; |
32 | | class VariantCanonicalScalarRef; |
33 | | struct VariantCanonicalScalarAdapter; |
34 | | |
35 | | bool canonical_equals(VariantCanonicalScalarRef left, VariantCanonicalScalarRef right); |
36 | | |
37 | | template <typename Sink> |
38 | | void canonical_hash(VariantCanonicalScalarRef value, Sink& sink); |
39 | | |
40 | | // Stack-only logical scalar view used by typed column adapters. String and binary values are |
41 | | // borrowed until the synchronous hash/serialization call returns; no column or DataType |
42 | | // dependency crosses into the codec. |
43 | | class VariantCanonicalScalarRef { |
44 | | public: |
45 | | enum class Kind : uint8_t { |
46 | | NULL_VALUE, |
47 | | BOOL, |
48 | | EXACT_INTEGER, |
49 | | DECIMAL, |
50 | | FLOATING, |
51 | | STRING, |
52 | | BINARY, |
53 | | DATE, |
54 | | TIMESTAMP_TZ, |
55 | | TIMESTAMP_NTZ, |
56 | | TIME, |
57 | | UUID, |
58 | | }; |
59 | | |
60 | | static VariantCanonicalScalarRef null_value() noexcept; |
61 | | static VariantCanonicalScalarRef boolean(bool value) noexcept; |
62 | | static VariantCanonicalScalarRef exact_integer(__int128 value); |
63 | | static VariantCanonicalScalarRef decimal(__int128 unscaled, uint8_t scale); |
64 | | static VariantCanonicalScalarRef float32(float value) noexcept; |
65 | | static VariantCanonicalScalarRef float64(double value) noexcept; |
66 | | static VariantCanonicalScalarRef string(StringRef value); |
67 | | static VariantCanonicalScalarRef binary(StringRef value); |
68 | | static VariantCanonicalScalarRef date(int32_t days_since_epoch) noexcept; |
69 | | static VariantCanonicalScalarRef timestamp_micros(int64_t value, bool utc_adjusted) noexcept; |
70 | | static VariantCanonicalScalarRef timestamp_nanos(int64_t value, bool utc_adjusted) noexcept; |
71 | | static VariantCanonicalScalarRef time_ntz_micros(int64_t value) noexcept; |
72 | | static VariantCanonicalScalarRef uuid(const std::array<uint8_t, 16>& value) noexcept; |
73 | | |
74 | | private: |
75 | 514 | explicit VariantCanonicalScalarRef(Kind kind) noexcept : _kind(kind) {} |
76 | | |
77 | | __int128 _integer = 0; |
78 | | double _floating = 0; |
79 | | StringRef _bytes; |
80 | | std::array<uint8_t, 16> _uuid {}; |
81 | | Kind _kind; |
82 | | uint8_t _scale = 0; |
83 | | bool _boolean = false; |
84 | | |
85 | | friend class CanonicalScalarSerializationPlan; |
86 | | friend bool canonical_equals(VariantCanonicalScalarRef left, VariantCanonicalScalarRef right); |
87 | | template <typename Sink> |
88 | | friend void canonical_hash(VariantCanonicalScalarRef value, Sink& sink); |
89 | | friend CanonicalScalarSerializationPlan prepare_canonical_serialize( |
90 | | VariantCanonicalScalarRef value); |
91 | | friend struct VariantCanonicalScalarAdapter; |
92 | | }; |
93 | | |
94 | | // Incremental adapters used by canonical_hash(). Each update consumes one canonical token. The |
95 | | // token traversal and token boundaries are shared by all sinks; the adapters only select the hash |
96 | | // family and carry its seed/state. |
97 | | class VariantXxHashSink { |
98 | | public: |
99 | 20.2k | explicit VariantXxHashSink(uint64_t seed = 0) noexcept : _state(seed) {} |
100 | | |
101 | | void update(const char* data, size_t size); |
102 | 20.2k | uint64_t digest() const noexcept { return _state; } |
103 | | |
104 | | private: |
105 | | uint64_t _state; |
106 | | }; |
107 | | |
108 | | class VariantCrc32HashSink { |
109 | | public: |
110 | 20.2k | explicit VariantCrc32HashSink(uint32_t seed = 0) noexcept : _state(seed) {} |
111 | | |
112 | | void update(const char* data, size_t size); |
113 | 20.2k | uint32_t digest() const noexcept { return _state; } |
114 | | |
115 | | private: |
116 | | uint32_t _state; |
117 | | }; |
118 | | |
119 | | class VariantCrc32cHashSink { |
120 | | public: |
121 | 20.2k | explicit VariantCrc32cHashSink(uint32_t seed = 0) noexcept : _state(seed) {} |
122 | | |
123 | | void update(const char* data, size_t size); |
124 | 20.2k | uint32_t digest() const noexcept { return _state; } |
125 | | |
126 | | private: |
127 | | uint32_t _state; |
128 | | }; |
129 | | |
130 | | // Compares logical Variant values. Numeric normalization is deliberately limited to the exact |
131 | | // integer domain that canonical Variant decimal16 can encode: [-(10^38-1), +(10^38-1)]. A finite |
132 | | // integral float/double outside that domain remains floating so canonical arena bytes stay valid. |
133 | | bool canonical_equals(VariantValueRef left, VariantValueRef right); |
134 | | |
135 | | // Hashes canonical logical tokens without re-encoding the value. Supported production sinks are |
136 | | // SipHash, VariantXxHashSink, VariantCrc32HashSink, and VariantCrc32cHashSink; all four are explicit |
137 | | // instantiations of the same traversal skeleton. |
138 | | template <typename Sink> |
139 | | void canonical_hash(VariantValueRef value, Sink& sink); |
140 | | |
141 | | extern template void canonical_hash<SipHash>(VariantValueRef value, SipHash& sink); |
142 | | extern template void canonical_hash<VariantXxHashSink>(VariantValueRef value, |
143 | | VariantXxHashSink& sink); |
144 | | extern template void canonical_hash<VariantCrc32HashSink>(VariantValueRef value, |
145 | | VariantCrc32HashSink& sink); |
146 | | extern template void canonical_hash<VariantCrc32cHashSink>(VariantValueRef value, |
147 | | VariantCrc32cHashSink& sink); |
148 | | |
149 | | extern template void canonical_hash<SipHash>(VariantCanonicalScalarRef value, SipHash& sink); |
150 | | extern template void canonical_hash<VariantXxHashSink>(VariantCanonicalScalarRef value, |
151 | | VariantXxHashSink& sink); |
152 | | extern template void canonical_hash<VariantCrc32HashSink>(VariantCanonicalScalarRef value, |
153 | | VariantCrc32HashSink& sink); |
154 | | extern template void canonical_hash<VariantCrc32cHashSink>(VariantCanonicalScalarRef value, |
155 | | VariantCrc32cHashSink& sink); |
156 | | |
157 | | // No-heap canonical arena plan for one scalar. The output layout is identical to |
158 | | // CanonicalSerializationPlan: [u32 payload_size][minimal metadata][canonical scalar]. |
159 | | class CanonicalScalarSerializationPlan { |
160 | | public: |
161 | 554 | size_t size() const noexcept { return _cell_size; } |
162 | | void write(char* destination, size_t capacity) const; |
163 | | |
164 | | private: |
165 | | CanonicalScalarSerializationPlan(VariantCanonicalScalarRef value, size_t cell_size) noexcept |
166 | 288 | : _value(value), _cell_size(cell_size) {} |
167 | | |
168 | | VariantCanonicalScalarRef _value; |
169 | | size_t _cell_size; |
170 | | |
171 | | friend CanonicalScalarSerializationPlan prepare_canonical_serialize( |
172 | | VariantCanonicalScalarRef value); |
173 | | }; |
174 | | |
175 | | CanonicalScalarSerializationPlan prepare_canonical_serialize(VariantCanonicalScalarRef value); |
176 | | |
177 | | // Owns a validated canonical serialization plan while borrowing the source Variant bytes. The |
178 | | // source metadata/value buffers must remain stable until write() returns. This lets column adapters |
179 | | // allocate their final contiguous storage exactly once without introducing an Arena dependency in |
180 | | // the codec. |
181 | | class CanonicalSerializationPlan { |
182 | | public: |
183 | | CanonicalSerializationPlan(CanonicalSerializationPlan&&) noexcept; |
184 | | CanonicalSerializationPlan& operator=(CanonicalSerializationPlan&&) noexcept; |
185 | | ~CanonicalSerializationPlan(); |
186 | | |
187 | | CanonicalSerializationPlan(const CanonicalSerializationPlan&) = delete; |
188 | | CanonicalSerializationPlan& operator=(const CanonicalSerializationPlan&) = delete; |
189 | | |
190 | | // Returns zero only for a moved-from plan. |
191 | | size_t size() const noexcept; |
192 | | |
193 | | // Writes exactly size() bytes. A null destination, moved-from plan, or insufficient capacity |
194 | | // fails before modifying caller-owned memory. |
195 | | void write(char* destination, size_t capacity) const; |
196 | | |
197 | | private: |
198 | | struct Impl; |
199 | | |
200 | | explicit CanonicalSerializationPlan(std::unique_ptr<Impl> impl) noexcept; |
201 | | |
202 | | std::unique_ptr<Impl> _impl; |
203 | | |
204 | | friend CanonicalSerializationPlan prepare_canonical_serialize(VariantValueRef value); |
205 | | }; |
206 | | |
207 | | // Validates and plans one canonical arena cell without allocating its final output bytes. |
208 | | CanonicalSerializationPlan prepare_canonical_serialize(VariantValueRef value); |
209 | | |
210 | | // Appends one canonical arena cell and returns the number of appended bytes. The layout is |
211 | | // [u32 payload_size][canonical_metadata][canonical_value], where payload_size excludes its own |
212 | | // four-byte prefix. On validation or size failure, destination is unchanged. This convenience |
213 | | // overload delegates to CanonicalSerializationPlan. |
214 | | size_t canonical_serialize(VariantValueRef value, std::string& destination); |
215 | | |
216 | | // Splits one bounded canonical arena cell without allocating or canonicalizing it. The returned |
217 | | // view borrows serialized and remains valid only while the input bytes remain stable. This checks |
218 | | // the exact payload prefix, derives the self-delimiting metadata size with bounded arithmetic, and |
219 | | // requires the remainder to contain exactly one complete Variant value. |
220 | | VariantValueRef parse_canonical_serialized(StringRef serialized); |
221 | | |
222 | | } // namespace doris |