/root/doris/contrib/faiss/faiss/utils/hamming-inl.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | | * |
4 | | * This source code is licensed under the MIT license found in the |
5 | | * LICENSE file in the root directory of this source tree. |
6 | | */ |
7 | | |
8 | | namespace faiss { |
9 | | |
10 | | // BitstringWriter and BitstringReader functions |
11 | | inline BitstringWriter::BitstringWriter(uint8_t* code, size_t code_size) |
12 | 0 | : code(code), code_size(code_size), i(0) { |
13 | 0 | memset(code, 0, code_size); |
14 | 0 | } |
15 | | |
16 | 0 | inline void BitstringWriter::write(uint64_t x, int nbit) { |
17 | 0 | assert(code_size * 8 >= nbit + i); |
18 | | // nb of available bits in i / 8 |
19 | 0 | int na = 8 - (i & 7); |
20 | |
|
21 | 0 | if (nbit <= na) { |
22 | 0 | code[i >> 3] |= x << (i & 7); |
23 | 0 | i += nbit; |
24 | 0 | return; |
25 | 0 | } else { |
26 | 0 | size_t j = i >> 3; |
27 | 0 | code[j++] |= x << (i & 7); |
28 | 0 | i += nbit; |
29 | 0 | x >>= na; |
30 | 0 | while (x != 0) { |
31 | 0 | code[j++] |= x; |
32 | 0 | x >>= 8; |
33 | 0 | } |
34 | 0 | } |
35 | 0 | } |
36 | | |
37 | | inline BitstringReader::BitstringReader(const uint8_t* code, size_t code_size) |
38 | 0 | : code(code), code_size(code_size), i(0) {} |
39 | | |
40 | 0 | inline uint64_t BitstringReader::read(int nbit) { |
41 | 0 | assert(code_size * 8 >= nbit + i); |
42 | | // nb of available bits in i / 8 |
43 | 0 | int na = 8 - (i & 7); |
44 | | // get available bits in current byte |
45 | 0 | uint64_t res = code[i >> 3] >> (i & 7); |
46 | 0 | if (nbit <= na) { |
47 | 0 | res &= (1 << nbit) - 1; |
48 | 0 | i += nbit; |
49 | 0 | return res; |
50 | 0 | } else { |
51 | 0 | int ofs = na; |
52 | 0 | size_t j = (i >> 3) + 1; |
53 | 0 | i += nbit; |
54 | 0 | nbit -= na; |
55 | 0 | while (nbit > 8) { |
56 | 0 | res |= ((uint64_t)code[j++]) << ofs; |
57 | 0 | ofs += 8; |
58 | 0 | nbit -= 8; // TODO remove nbit |
59 | 0 | } |
60 | 0 | uint64_t last_byte = code[j]; |
61 | 0 | last_byte &= (1 << nbit) - 1; |
62 | 0 | res |= last_byte << ofs; |
63 | 0 | return res; |
64 | 0 | } |
65 | 0 | } |
66 | | |
67 | | /** This class maintains a list of best distances seen so far. |
68 | | * |
69 | | * Since the distances are in a limited range (0 to nbit), the |
70 | | * object maintains one list per possible distance, and fills |
71 | | * in only the n-first lists, such that the sum of sizes of the |
72 | | * n lists is below k. |
73 | | */ |
74 | | template <class HammingComputer> |
75 | | struct HCounterState { |
76 | | int* counters; |
77 | | int64_t* ids_per_dis; |
78 | | |
79 | | HammingComputer hc; |
80 | | int thres; |
81 | | int count_lt; |
82 | | int count_eq; |
83 | | int k; |
84 | | |
85 | | HCounterState( |
86 | | int* counters, |
87 | | int64_t* ids_per_dis, |
88 | | const uint8_t* x, |
89 | | int d, |
90 | | int k) |
91 | 0 | : counters(counters), |
92 | 0 | ids_per_dis(ids_per_dis), |
93 | 0 | hc(x, d / 8), |
94 | 0 | thres(d + 1), |
95 | 0 | count_lt(0), |
96 | 0 | count_eq(0), |
97 | 0 | k(k) {} Unexecuted instantiation: _ZN5faiss13HCounterStateINS_16HammingComputer4EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_16HammingComputer8EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer16EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer20EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer32EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer64EEC2EPiPlPKhii Unexecuted instantiation: _ZN5faiss13HCounterStateINS_22HammingComputerDefaultEEC2EPiPlPKhii |
98 | | |
99 | 0 | void update_counter(const uint8_t* y, size_t j) { |
100 | 0 | int32_t dis = hc.hamming(y); |
101 | |
|
102 | 0 | if (dis <= thres) { |
103 | 0 | if (dis < thres) { |
104 | 0 | ids_per_dis[dis * k + counters[dis]++] = j; |
105 | 0 | ++count_lt; |
106 | 0 | while (count_lt == k && thres > 0) { |
107 | 0 | --thres; |
108 | 0 | count_eq = counters[thres]; |
109 | 0 | count_lt -= count_eq; |
110 | 0 | } |
111 | 0 | } else if (count_eq < k) { |
112 | 0 | ids_per_dis[dis * k + count_eq++] = j; |
113 | 0 | counters[dis] = count_eq; |
114 | 0 | } |
115 | 0 | } |
116 | 0 | } Unexecuted instantiation: _ZN5faiss13HCounterStateINS_16HammingComputer4EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_16HammingComputer8EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer16EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer20EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer32EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_17HammingComputer64EE14update_counterEPKhm Unexecuted instantiation: _ZN5faiss13HCounterStateINS_22HammingComputerDefaultEE14update_counterEPKhm |
117 | | }; |
118 | | |
119 | | } // namespace faiss |