Coverage Report

Created: 2026-07-14 14:34

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/variant/variant_metadata.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/variant/variant_metadata.h"
19
20
#include <cstdint>
21
22
#include "common/exception.h"
23
#include "util/variant/variant_encoding.h"
24
25
namespace doris {
26
namespace {
27
28
10.7M
uint32_t read_unsigned(const char* data, uint8_t width) {
29
10.7M
    uint32_t result = 0;
30
41.9M
    for (uint8_t i = 0; i < width; ++i) {
31
31.1M
        result |= static_cast<uint32_t>(static_cast<uint8_t>(data[i])) << (i * 8);
32
31.1M
    }
33
10.7M
    return result;
34
10.7M
}
35
36
7.54M
void require_bytes(size_t available, size_t required, const char* field) {
37
7.54M
    if (available < required) {
38
12
        throw Exception(ErrorCode::CORRUPTION,
39
12
                        "Truncated Variant metadata while reading {}: need {} bytes, have {}",
40
12
                        field, required, available);
41
12
    }
42
7.54M
}
43
44
} // namespace
45
46
270
uint8_t VariantMetadataRef::version() const {
47
270
    require_bytes(size, 1, "header");
48
270
    return static_cast<uint8_t>(data[0]) & VARIANT_METADATA_VERSION_MASK;
49
270
}
50
51
1.00M
bool VariantMetadataRef::sorted_strings() const {
52
1.00M
    require_bytes(size, 1, "header");
53
1.00M
    return (static_cast<uint8_t>(data[0]) & VARIANT_METADATA_SORTED_STRINGS_MASK) != 0;
54
1.00M
}
55
56
242
uint8_t VariantMetadataRef::offset_size() const {
57
242
    require_bytes(size, 1, "header");
58
242
    return static_cast<uint8_t>(
59
242
            ((static_cast<uint8_t>(data[0]) >> VARIANT_METADATA_OFFSET_SIZE_SHIFT) &
60
242
             VARIANT_METADATA_OFFSET_SIZE_MASK) +
61
242
            1);
62
242
}
63
64
2.17M
VariantMetadataRef::Layout VariantMetadataRef::_layout() const {
65
2.17M
    require_bytes(size, 1, "header");
66
2.17M
    const auto header = static_cast<uint8_t>(data[0]);
67
2.17M
    const uint8_t metadata_version = header & VARIANT_METADATA_VERSION_MASK;
68
2.17M
    if (metadata_version != VARIANT_ENCODING_VERSION) {
69
5
        throw Exception(ErrorCode::INVALID_ARGUMENT,
70
5
                        "Unsupported Variant metadata version {}, expected {}", metadata_version,
71
5
                        VARIANT_ENCODING_VERSION);
72
5
    }
73
74
2.17M
    const auto width = static_cast<uint8_t>(
75
2.17M
            ((header >> VARIANT_METADATA_OFFSET_SIZE_SHIFT) & VARIANT_METADATA_OFFSET_SIZE_MASK) +
76
2.17M
            1);
77
2.17M
    require_bytes(size - 1, width, "dictionary size");
78
2.17M
    const uint32_t count = read_unsigned(data + 1, width);
79
80
2.17M
    constexpr size_t HEADER_SIZE = 1;
81
2.17M
    const size_t dictionary_size_offset = HEADER_SIZE;
82
2.17M
    const size_t offsets_offset = dictionary_size_offset + width;
83
2.17M
    const size_t offsets_bytes = (static_cast<size_t>(count) + 1) * width;
84
2.17M
    require_bytes(size - offsets_offset, offsets_bytes, "dictionary offsets");
85
2.17M
    const size_t strings_offset = offsets_offset + offsets_bytes;
86
87
2.17M
    const uint32_t first_offset = read_unsigned(data + offsets_offset, width);
88
2.17M
    if (first_offset != 0) {
89
0
        throw Exception(ErrorCode::CORRUPTION,
90
0
                        "Invalid Variant metadata first dictionary offset {}", first_offset);
91
0
    }
92
2.17M
    const uint32_t final_offset =
93
2.17M
            read_unsigned(data + offsets_offset + static_cast<size_t>(count) * width, width);
94
2.17M
    if (final_offset != size - strings_offset) {
95
2
        throw Exception(ErrorCode::CORRUPTION,
96
2
                        "Invalid Variant metadata final dictionary offset {} for {} bytes",
97
2
                        final_offset, size - strings_offset);
98
2
    }
99
2.17M
    return {.offset_width = width,
100
2.17M
            .num_keys = count,
101
2.17M
            .offsets_offset = offsets_offset,
102
2.17M
            .strings_offset = strings_offset};
103
2.17M
}
104
105
53.5k
uint32_t VariantMetadataRef::dict_size() const {
106
53.5k
    return _layout().num_keys;
107
53.5k
}
108
109
2.12M
StringRef VariantMetadataRef::key_at(uint32_t id) const {
110
2.12M
    const Layout layout = _layout();
111
2.12M
    if (id >= layout.num_keys) {
112
1
        throw Exception(ErrorCode::INVALID_ARGUMENT,
113
1
                        "Variant metadata dictionary id {} is out of range [0, {})", id,
114
1
                        layout.num_keys);
115
1
    }
116
2.12M
    const uint32_t begin = read_unsigned(
117
2.12M
            data + layout.offsets_offset + static_cast<size_t>(id) * layout.offset_width,
118
2.12M
            layout.offset_width);
119
2.12M
    const uint32_t end = read_unsigned(
120
2.12M
            data + layout.offsets_offset + (static_cast<size_t>(id) + 1) * layout.offset_width,
121
2.12M
            layout.offset_width);
122
2.12M
    const size_t strings_size = size - layout.strings_offset;
123
2.12M
    if (begin > end || end > strings_size) {
124
1
        throw Exception(ErrorCode::CORRUPTION,
125
1
                        "Invalid Variant metadata dictionary offsets [{}, {}) for {} bytes", begin,
126
1
                        end, strings_size);
127
1
    }
128
2.12M
    return {data + layout.strings_offset + begin, end - begin};
129
2.12M
}
130
131
703
int64_t VariantMetadataRef::find_key(StringRef key) const {
132
703
    const Layout layout = _layout();
133
703
    if (!sorted_strings()) {
134
66.0k
        for (uint32_t id = 0; id < layout.num_keys; ++id) {
135
66.0k
            if (key_at(id) == key) {
136
535
                return id;
137
535
            }
138
66.0k
        }
139
1
        return -1;
140
536
    }
141
142
167
    uint32_t begin = 0;
143
167
    uint32_t end = layout.num_keys;
144
608
    while (begin < end) {
145
441
        const uint32_t middle = begin + (end - begin) / 2;
146
441
        const int comparison = key_at(middle).compare(key);
147
441
        if (comparison < 0) {
148
167
            begin = middle + 1;
149
274
        } else {
150
274
            end = middle;
151
274
        }
152
441
    }
153
167
    if (begin < layout.num_keys && key_at(begin) == key) {
154
153
        return begin;
155
153
    }
156
14
    return -1;
157
167
}
158
159
1.31k
void VariantMetadataRef::validate() const {
160
1.31k
    const Layout layout = _layout();
161
1.31k
    StringRef previous;
162
1.00M
    for (uint32_t id = 0; id < layout.num_keys; ++id) {
163
1.00M
        const StringRef current = key_at(id);
164
1.00M
        if (sorted_strings() && id != 0 && previous.compare(current) >= 0) {
165
2
            throw Exception(ErrorCode::CORRUPTION,
166
2
                            "Variant metadata dictionary is not sorted and unique at id {}", id);
167
2
        }
168
1.00M
        previous = current;
169
1.00M
    }
170
1.31k
}
171
172
} // namespace doris