be/src/util/variant/variant_metadata.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "util/variant/variant_metadata.h" |
19 | | |
20 | | #include <cstdint> |
21 | | |
22 | | #include "common/exception.h" |
23 | | #include "util/variant/variant_encoding.h" |
24 | | |
25 | | namespace doris { |
26 | | namespace { |
27 | | |
28 | 10.7M | uint32_t read_unsigned(const char* data, uint8_t width) { |
29 | 10.7M | uint32_t result = 0; |
30 | 41.9M | for (uint8_t i = 0; i < width; ++i) { |
31 | 31.1M | result |= static_cast<uint32_t>(static_cast<uint8_t>(data[i])) << (i * 8); |
32 | 31.1M | } |
33 | 10.7M | return result; |
34 | 10.7M | } |
35 | | |
36 | 7.54M | void require_bytes(size_t available, size_t required, const char* field) { |
37 | 7.54M | if (available < required) { |
38 | 12 | throw Exception(ErrorCode::CORRUPTION, |
39 | 12 | "Truncated Variant metadata while reading {}: need {} bytes, have {}", |
40 | 12 | field, required, available); |
41 | 12 | } |
42 | 7.54M | } |
43 | | |
44 | | } // namespace |
45 | | |
46 | 270 | uint8_t VariantMetadataRef::version() const { |
47 | 270 | require_bytes(size, 1, "header"); |
48 | 270 | return static_cast<uint8_t>(data[0]) & VARIANT_METADATA_VERSION_MASK; |
49 | 270 | } |
50 | | |
51 | 1.00M | bool VariantMetadataRef::sorted_strings() const { |
52 | 1.00M | require_bytes(size, 1, "header"); |
53 | 1.00M | return (static_cast<uint8_t>(data[0]) & VARIANT_METADATA_SORTED_STRINGS_MASK) != 0; |
54 | 1.00M | } |
55 | | |
56 | 242 | uint8_t VariantMetadataRef::offset_size() const { |
57 | 242 | require_bytes(size, 1, "header"); |
58 | 242 | return static_cast<uint8_t>( |
59 | 242 | ((static_cast<uint8_t>(data[0]) >> VARIANT_METADATA_OFFSET_SIZE_SHIFT) & |
60 | 242 | VARIANT_METADATA_OFFSET_SIZE_MASK) + |
61 | 242 | 1); |
62 | 242 | } |
63 | | |
64 | 2.17M | VariantMetadataRef::Layout VariantMetadataRef::_layout() const { |
65 | 2.17M | require_bytes(size, 1, "header"); |
66 | 2.17M | const auto header = static_cast<uint8_t>(data[0]); |
67 | 2.17M | const uint8_t metadata_version = header & VARIANT_METADATA_VERSION_MASK; |
68 | 2.17M | if (metadata_version != VARIANT_ENCODING_VERSION) { |
69 | 5 | throw Exception(ErrorCode::INVALID_ARGUMENT, |
70 | 5 | "Unsupported Variant metadata version {}, expected {}", metadata_version, |
71 | 5 | VARIANT_ENCODING_VERSION); |
72 | 5 | } |
73 | | |
74 | 2.17M | const auto width = static_cast<uint8_t>( |
75 | 2.17M | ((header >> VARIANT_METADATA_OFFSET_SIZE_SHIFT) & VARIANT_METADATA_OFFSET_SIZE_MASK) + |
76 | 2.17M | 1); |
77 | 2.17M | require_bytes(size - 1, width, "dictionary size"); |
78 | 2.17M | const uint32_t count = read_unsigned(data + 1, width); |
79 | | |
80 | 2.17M | constexpr size_t HEADER_SIZE = 1; |
81 | 2.17M | const size_t dictionary_size_offset = HEADER_SIZE; |
82 | 2.17M | const size_t offsets_offset = dictionary_size_offset + width; |
83 | 2.17M | const size_t offsets_bytes = (static_cast<size_t>(count) + 1) * width; |
84 | 2.17M | require_bytes(size - offsets_offset, offsets_bytes, "dictionary offsets"); |
85 | 2.17M | const size_t strings_offset = offsets_offset + offsets_bytes; |
86 | | |
87 | 2.17M | const uint32_t first_offset = read_unsigned(data + offsets_offset, width); |
88 | 2.17M | if (first_offset != 0) { |
89 | 0 | throw Exception(ErrorCode::CORRUPTION, |
90 | 0 | "Invalid Variant metadata first dictionary offset {}", first_offset); |
91 | 0 | } |
92 | 2.17M | const uint32_t final_offset = |
93 | 2.17M | read_unsigned(data + offsets_offset + static_cast<size_t>(count) * width, width); |
94 | 2.17M | if (final_offset != size - strings_offset) { |
95 | 2 | throw Exception(ErrorCode::CORRUPTION, |
96 | 2 | "Invalid Variant metadata final dictionary offset {} for {} bytes", |
97 | 2 | final_offset, size - strings_offset); |
98 | 2 | } |
99 | 2.17M | return {.offset_width = width, |
100 | 2.17M | .num_keys = count, |
101 | 2.17M | .offsets_offset = offsets_offset, |
102 | 2.17M | .strings_offset = strings_offset}; |
103 | 2.17M | } |
104 | | |
105 | 53.5k | uint32_t VariantMetadataRef::dict_size() const { |
106 | 53.5k | return _layout().num_keys; |
107 | 53.5k | } |
108 | | |
109 | 2.12M | StringRef VariantMetadataRef::key_at(uint32_t id) const { |
110 | 2.12M | const Layout layout = _layout(); |
111 | 2.12M | if (id >= layout.num_keys) { |
112 | 1 | throw Exception(ErrorCode::INVALID_ARGUMENT, |
113 | 1 | "Variant metadata dictionary id {} is out of range [0, {})", id, |
114 | 1 | layout.num_keys); |
115 | 1 | } |
116 | 2.12M | const uint32_t begin = read_unsigned( |
117 | 2.12M | data + layout.offsets_offset + static_cast<size_t>(id) * layout.offset_width, |
118 | 2.12M | layout.offset_width); |
119 | 2.12M | const uint32_t end = read_unsigned( |
120 | 2.12M | data + layout.offsets_offset + (static_cast<size_t>(id) + 1) * layout.offset_width, |
121 | 2.12M | layout.offset_width); |
122 | 2.12M | const size_t strings_size = size - layout.strings_offset; |
123 | 2.12M | if (begin > end || end > strings_size) { |
124 | 1 | throw Exception(ErrorCode::CORRUPTION, |
125 | 1 | "Invalid Variant metadata dictionary offsets [{}, {}) for {} bytes", begin, |
126 | 1 | end, strings_size); |
127 | 1 | } |
128 | 2.12M | return {data + layout.strings_offset + begin, end - begin}; |
129 | 2.12M | } |
130 | | |
131 | 703 | int64_t VariantMetadataRef::find_key(StringRef key) const { |
132 | 703 | const Layout layout = _layout(); |
133 | 703 | if (!sorted_strings()) { |
134 | 66.0k | for (uint32_t id = 0; id < layout.num_keys; ++id) { |
135 | 66.0k | if (key_at(id) == key) { |
136 | 535 | return id; |
137 | 535 | } |
138 | 66.0k | } |
139 | 1 | return -1; |
140 | 536 | } |
141 | | |
142 | 167 | uint32_t begin = 0; |
143 | 167 | uint32_t end = layout.num_keys; |
144 | 608 | while (begin < end) { |
145 | 441 | const uint32_t middle = begin + (end - begin) / 2; |
146 | 441 | const int comparison = key_at(middle).compare(key); |
147 | 441 | if (comparison < 0) { |
148 | 167 | begin = middle + 1; |
149 | 274 | } else { |
150 | 274 | end = middle; |
151 | 274 | } |
152 | 441 | } |
153 | 167 | if (begin < layout.num_keys && key_at(begin) == key) { |
154 | 153 | return begin; |
155 | 153 | } |
156 | 14 | return -1; |
157 | 167 | } |
158 | | |
159 | 1.31k | void VariantMetadataRef::validate() const { |
160 | 1.31k | const Layout layout = _layout(); |
161 | 1.31k | StringRef previous; |
162 | 1.00M | for (uint32_t id = 0; id < layout.num_keys; ++id) { |
163 | 1.00M | const StringRef current = key_at(id); |
164 | 1.00M | if (sorted_strings() && id != 0 && previous.compare(current) >= 0) { |
165 | 2 | throw Exception(ErrorCode::CORRUPTION, |
166 | 2 | "Variant metadata dictionary is not sorted and unique at id {}", id); |
167 | 2 | } |
168 | 1.00M | previous = current; |
169 | 1.00M | } |
170 | 1.31k | } |
171 | | |
172 | | } // namespace doris |