/root/doris/contrib/faiss/faiss/impl/io.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | | * |
4 | | * This source code is licensed under the MIT license found in the |
5 | | * LICENSE file in the root directory of this source tree. |
6 | | */ |
7 | | |
8 | | // -*- c++ -*- |
9 | | |
10 | | /*********************************************************** |
11 | | * Abstract I/O objects |
12 | | * |
13 | | * I/O is always sequential, seek does not need to be supported |
14 | | * (indexes could be read or written to a pipe). |
15 | | ***********************************************************/ |
16 | | |
17 | | #pragma once |
18 | | |
19 | | #include <cstddef> |
20 | | #include <cstdint> |
21 | | #include <cstdio> |
22 | | #include <string> |
23 | | #include <vector> |
24 | | |
25 | | namespace faiss { |
26 | | |
27 | | struct IOReader { |
28 | | // name that can be used in error messages |
29 | | std::string name; |
30 | | |
31 | | // fread. Returns number of items read or 0 in case of EOF. |
32 | | virtual size_t operator()(void* ptr, size_t size, size_t nitems) = 0; |
33 | | |
34 | | // return a file number that can be memory-mapped |
35 | | virtual int filedescriptor(); |
36 | | |
37 | 17 | virtual ~IOReader() {} |
38 | | }; |
39 | | |
40 | | struct IOWriter { |
41 | | // name that can be used in error messages |
42 | | std::string name; |
43 | | |
44 | | // fwrite. Return number of items written |
45 | | virtual size_t operator()(const void* ptr, size_t size, size_t nitems) = 0; |
46 | | |
47 | | // return a file number that can be memory-mapped |
48 | | virtual int filedescriptor(); |
49 | | |
50 | 19 | virtual ~IOWriter() noexcept(false) {} |
51 | | }; |
52 | | |
53 | | struct VectorIOReader : IOReader { |
54 | | std::vector<uint8_t> data; |
55 | | size_t rp = 0; |
56 | | size_t operator()(void* ptr, size_t size, size_t nitems) override; |
57 | | }; |
58 | | |
59 | | struct VectorIOWriter : IOWriter { |
60 | | std::vector<uint8_t> data; |
61 | | size_t operator()(const void* ptr, size_t size, size_t nitems) override; |
62 | | }; |
63 | | |
64 | | struct FileIOReader : IOReader { |
65 | | FILE* f = nullptr; |
66 | | bool need_close = false; |
67 | | |
68 | | FileIOReader(FILE* rf); |
69 | | |
70 | | FileIOReader(const char* fname); |
71 | | |
72 | | ~FileIOReader() override; |
73 | | |
74 | | size_t operator()(void* ptr, size_t size, size_t nitems) override; |
75 | | |
76 | | int filedescriptor() override; |
77 | | }; |
78 | | |
79 | | struct FileIOWriter : IOWriter { |
80 | | FILE* f = nullptr; |
81 | | bool need_close = false; |
82 | | |
83 | | FileIOWriter(FILE* wf); |
84 | | |
85 | | FileIOWriter(const char* fname); |
86 | | |
87 | | ~FileIOWriter() override; |
88 | | |
89 | | size_t operator()(const void* ptr, size_t size, size_t nitems) override; |
90 | | |
91 | | int filedescriptor() override; |
92 | | }; |
93 | | |
94 | | /******************************************************* |
95 | | * Buffered reader + writer |
96 | | * |
97 | | * They attempt to read and write only buffers of size bsz to the |
98 | | * underlying reader or writer. This is done by splitting or merging |
99 | | * the read/write functions. |
100 | | *******************************************************/ |
101 | | |
102 | | /** wraps an ioreader to make buffered reads to avoid too small reads */ |
103 | | struct BufferedIOReader : IOReader { |
104 | | IOReader* reader; |
105 | | size_t bsz; |
106 | | size_t ofs; ///< offset in input stream |
107 | | size_t ofs2; ///< number of bytes returned to caller |
108 | | size_t b0, b1; ///< range of available bytes in the buffer |
109 | | std::vector<char> buffer; |
110 | | |
111 | | /** |
112 | | * @param bsz buffer size (bytes). Reads will be done by batched of |
113 | | * this size |
114 | | */ |
115 | | explicit BufferedIOReader(IOReader* reader, size_t bsz = 1024 * 1024); |
116 | | |
117 | | size_t operator()(void* ptr, size_t size, size_t nitems) override; |
118 | | }; |
119 | | |
120 | | struct BufferedIOWriter : IOWriter { |
121 | | IOWriter* writer; |
122 | | size_t bsz; |
123 | | size_t ofs; |
124 | | size_t ofs2; ///< number of bytes received from caller |
125 | | size_t b0; ///< amount of data in buffer |
126 | | std::vector<char> buffer; |
127 | | |
128 | | explicit BufferedIOWriter(IOWriter* writer, size_t bsz = 1024 * 1024); |
129 | | |
130 | | size_t operator()(const void* ptr, size_t size, size_t nitems) override; |
131 | | |
132 | | // flushes |
133 | | ~BufferedIOWriter() override; |
134 | | }; |
135 | | |
136 | | /// cast a 4-character string to a uint32_t that can be written and read easily |
137 | | uint32_t fourcc(const char sx[4]); |
138 | | uint32_t fourcc(const std::string& sx); |
139 | | |
140 | | // decoding of fourcc (int32 -> string) |
141 | | void fourcc_inv(uint32_t x, char str[5]); |
142 | | std::string fourcc_inv(uint32_t x); |
143 | | std::string fourcc_inv_printable(uint32_t x); |
144 | | |
145 | | } // namespace faiss |