be/src/util/simd/selection.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <cstdint> |
21 | | #include <cstring> |
22 | | |
23 | | #include "common/compiler_util.h" |
24 | | #include "util/simd/bits.h" |
25 | | |
26 | | namespace doris::simd { |
27 | | |
28 | | #if defined(__x86_64__) |
29 | | namespace detail { |
30 | | struct alignas(16) SelectionLookupTable { |
31 | | uint16_t positions[256][8] {}; |
32 | | uint8_t counts[256] {}; |
33 | | |
34 | 0 | constexpr SelectionLookupTable() { |
35 | 0 | for (unsigned mask = 0; mask < 256; ++mask) { |
36 | 0 | for (unsigned bit = 0; bit < 8; ++bit) { |
37 | 0 | if (mask & (1U << bit)) { |
38 | 0 | positions[mask][counts[mask]++] = static_cast<uint16_t>(bit); |
39 | 0 | } |
40 | 0 | } |
41 | 0 | } |
42 | 0 | } |
43 | | }; |
44 | | |
45 | | inline constexpr SelectionLookupTable SELECTION_LOOKUP_TABLE; |
46 | | } // namespace detail |
47 | | #endif |
48 | | |
49 | | // Append base + bit positions in ascending order. The caller must provide room for |
50 | | // bits_mask_length() entries, and base + bits_mask_length() - 1 must fit in uint16_t. |
51 | | // Only the returned number of entries are valid; the lookup path can write extra entries. |
52 | | ALWAYS_INLINE inline uint16_t append_selected_indices( |
53 | 940k | decltype(bytes_mask_to_bits_mask(nullptr)) mask, uint16_t base, uint16_t* output) { |
54 | 940k | uint16_t count = 0; |
55 | 940k | #if defined(__x86_64__) |
56 | 4.70M | for (unsigned offset = 0; offset < 32; offset += 8) { |
57 | 3.76M | const auto part = static_cast<uint8_t>(mask >> offset); |
58 | 3.76M | uint16_t indices[8]; |
59 | 33.7M | for (unsigned lane = 0; lane < 8; ++lane) { |
60 | 30.0M | indices[lane] = static_cast<uint16_t>( |
61 | 30.0M | base + offset + detail::SELECTION_LOOKUP_TABLE.positions[part][lane]); |
62 | 30.0M | } |
63 | | // count <= offset, so the fixed-width store stays within 32 output entries. |
64 | 3.76M | std::memcpy(output + count, indices, sizeof(indices)); |
65 | 3.76M | count += detail::SELECTION_LOOKUP_TABLE.counts[part]; |
66 | 3.76M | } |
67 | | #else |
68 | | iterate_through_bits_mask( |
69 | | [&](const int bit_pos) { output[count++] = base + static_cast<uint16_t>(bit_pos); }, |
70 | | mask); |
71 | | #endif |
72 | 940k | return count; |
73 | 940k | } |
74 | | |
75 | | } // namespace doris::simd |