contrib/faiss/faiss/invlists/DirectMap.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | | * |
4 | | * This source code is licensed under the MIT license found in the |
5 | | * LICENSE file in the root directory of this source tree. |
6 | | */ |
7 | | |
8 | | // -*- c++ -*- |
9 | | |
10 | | #ifndef FAISS_DIRECT_MAP_H |
11 | | #define FAISS_DIRECT_MAP_H |
12 | | |
13 | | #include <faiss/invlists/InvertedLists.h> |
14 | | #include <unordered_map> |
15 | | |
16 | | namespace faiss { |
17 | | |
18 | | struct IDSelector; |
19 | | |
20 | | // When offsets list id + offset are encoded in an uint64 |
21 | | // we call this LO = list-offset |
22 | | |
23 | 0 | inline uint64_t lo_build(uint64_t list_id, uint64_t offset) { |
24 | 0 | return list_id << 32 | offset; |
25 | 0 | } |
26 | | |
27 | 0 | inline uint64_t lo_listno(uint64_t lo) { |
28 | 0 | return lo >> 32; |
29 | 0 | } |
30 | | |
31 | 0 | inline uint64_t lo_offset(uint64_t lo) { |
32 | 0 | return lo & 0xffffffff; |
33 | 0 | } |
34 | | |
35 | | /** |
36 | | * Direct map: a way to map back from ids to inverted lists |
37 | | */ |
38 | | struct DirectMap { |
39 | | enum Type { |
40 | | NoMap = 0, // default |
41 | | Array = 1, // sequential ids (only for add, no add_with_ids) |
42 | | Hashtable = 2 // arbitrary ids |
43 | | }; |
44 | | Type type; |
45 | | |
46 | | /// map for direct access to the elements. Map ids to LO-encoded entries. |
47 | | std::vector<idx_t> array; |
48 | | std::unordered_map<idx_t, idx_t> hashtable; |
49 | | |
50 | | DirectMap(); |
51 | | |
52 | | /// set type and initialize |
53 | | void set_type(Type new_type, const InvertedLists* invlists, size_t ntotal); |
54 | | |
55 | | /// get an entry |
56 | | idx_t get(idx_t id) const; |
57 | | |
58 | | /// for quick checks |
59 | 0 | bool no() const { |
60 | 0 | return type == NoMap; |
61 | 0 | } |
62 | | |
63 | | /** |
64 | | * update the direct_map |
65 | | */ |
66 | | |
67 | | /// throw if Array and ids is not NULL |
68 | | void check_can_add(const idx_t* ids); |
69 | | |
70 | | /// non thread-safe version |
71 | | void add_single_id(idx_t id, idx_t list_no, size_t offset); |
72 | | |
73 | | /// remove all entries |
74 | | void clear(); |
75 | | |
76 | | /** |
77 | | * operations on inverted lists that require translation with a DirectMap |
78 | | */ |
79 | | |
80 | | /// remove ids from the InvertedLists, possibly using the direct map |
81 | | size_t remove_ids(const IDSelector& sel, InvertedLists* invlists); |
82 | | |
83 | | /// update entries, using the direct map |
84 | | void update_codes( |
85 | | InvertedLists* invlists, |
86 | | int n, |
87 | | const idx_t* ids, |
88 | | const idx_t* list_nos, |
89 | | const uint8_t* codes); |
90 | | }; |
91 | | |
92 | | /// Thread-safe way of updating the direct_map |
93 | | struct DirectMapAdd { |
94 | | using Type = DirectMap::Type; |
95 | | |
96 | | DirectMap& direct_map; |
97 | | DirectMap::Type type; |
98 | | size_t ntotal; |
99 | | size_t n; |
100 | | const idx_t* xids; |
101 | | |
102 | | std::vector<idx_t> all_ofs; |
103 | | |
104 | | DirectMapAdd(DirectMap& direct_map, size_t n, const idx_t* xids); |
105 | | |
106 | | /// add vector i (with id xids[i]) at list_no and offset |
107 | | void add(size_t i, idx_t list_no, size_t offset); |
108 | | |
109 | | ~DirectMapAdd(); |
110 | | }; |
111 | | |
112 | | } // namespace faiss |
113 | | |
114 | | #endif |