be/src/io/fs/obj_storage_client.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <optional> |
21 | | |
22 | | #include "io/fs/file_system.h" |
23 | | #include "io/fs/path.h" |
24 | | namespace doris { |
25 | | class Status; |
26 | | struct S3ClientConf; |
27 | | namespace io { |
28 | | |
29 | | // Names are in lexico order. |
30 | | enum class ObjStorageType : uint8_t { |
31 | | UNKNOWN = 0, |
32 | | AWS = 1, |
33 | | AZURE, |
34 | | BOS, |
35 | | COS, |
36 | | OSS, |
37 | | OBS, |
38 | | GCP, |
39 | | TOS, |
40 | | }; |
41 | | |
42 | | struct ObjectStoragePathOptions { |
43 | | Path path = ""; |
44 | | std::string bucket = std::string(); // blob container in azure |
45 | | std::string key = std::string(); // blob name in azure |
46 | | std::string prefix = std::string(); // for batch delete and recursive delete |
47 | | std::optional<std::string> upload_id = std::nullopt; // only used for S3 upload |
48 | | }; |
49 | | |
50 | | struct ObjectCompleteMultiPart { |
51 | | int part_num = 0; |
52 | | std::string etag = std::string(); |
53 | | }; |
54 | | |
55 | | struct ObjectStorageStatus { |
56 | | int code = 0; |
57 | | std::string msg = std::string(); |
58 | | }; |
59 | | |
60 | | // We only store error code along with err_msg instead of Status to unify BE and recycler's error handle logic |
61 | | struct ObjectStorageResponse { |
62 | | ObjectStorageStatus status {}; |
63 | | int http_code {200}; |
64 | | std::string request_id = std::string(); |
65 | 2.23k | static ObjectStorageResponse OK() { |
66 | | // clang-format off |
67 | 2.23k | return { |
68 | 2.23k | .status { .code = 0, }, |
69 | 2.23k | .http_code = 200, |
70 | 2.23k | }; |
71 | | // clang-format on |
72 | 2.23k | } |
73 | | }; |
74 | | |
75 | | struct ObjectStorageUploadResponse { |
76 | | ObjectStorageResponse resp {}; |
77 | | std::optional<std::string> upload_id = std::nullopt; |
78 | | std::optional<std::string> etag = std::nullopt; |
79 | | }; |
80 | | |
81 | | struct ObjectStorageHeadResponse : ObjectStorageResponse { |
82 | | ObjectStorageResponse resp {}; |
83 | | long long file_size {0}; |
84 | | }; |
85 | | |
86 | | class ObjStorageClient { |
87 | | public: |
88 | 66 | virtual ~ObjStorageClient() = default; |
89 | | // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. |
90 | | // The input parameters should include the bucket and key for the object storage. |
91 | | virtual ObjectStorageUploadResponse create_multipart_upload( |
92 | | const ObjectStoragePathOptions& opts) = 0; |
93 | | // To directly upload a piece of data to object storage and generate a user-visible file. |
94 | | // You need to clearly specify the bucket and key |
95 | | virtual ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, |
96 | | std::string_view stream) = 0; |
97 | | // To upload a part of a large file to object storage as a temporary file, which is not visible to the user |
98 | | // The temporary file's ID is the value of the part_num passed in |
99 | | // You need to specify the bucket and key along with the upload_id if it's AWS-compatible system |
100 | | // For the same bucket and key, as well as the same part_num, it will directly replace the original temporary file. |
101 | | virtual ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, |
102 | | std::string_view stream, int part_num) = 0; |
103 | | // To combine the previously uploaded multiple file parts into a complete file, the file name is the name of the key passed in. |
104 | | // If it is an AWS-compatible system, the upload_id needs to be included. |
105 | | // After a successful execution, the large file can be accessed in the object storage |
106 | | virtual ObjectStorageResponse complete_multipart_upload( |
107 | | const ObjectStoragePathOptions& opts, |
108 | | const std::vector<ObjectCompleteMultiPart>& completed_parts) = 0; |
109 | | // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. |
110 | | // If it exists, it will return the corresponding file size |
111 | | virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; |
112 | | // According to the bucket and key, it finds the corresponding file in the object storage |
113 | | // and starting from the offset, it reads bytes_read bytes into the buffer, with size_return recording the actual number of bytes read |
114 | | virtual ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, |
115 | | size_t offset, size_t bytes_read, |
116 | | size_t* size_return) = 0; |
117 | | // According to the passed bucket and prefix, it traverses and retrieves all files under the prefix, and returns the name and file size of all files. |
118 | | // **Notice**: The files returned by this function contains the full key in object storage. |
119 | | virtual ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, |
120 | | std::vector<FileInfo>* files) = 0; |
121 | | // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. |
122 | | virtual ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, |
123 | | std::vector<std::string> objs) = 0; |
124 | | // Delete the file named key in the object storage bucket. |
125 | | virtual ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) = 0; |
126 | | // According to the prefix, recursively delete all files under the prefix. |
127 | | virtual ObjectStorageResponse delete_objects_recursively( |
128 | | const ObjectStoragePathOptions& opts) = 0; |
129 | | // Return a presigned URL for users to access the object |
130 | | virtual std::string generate_presigned_url(const ObjectStoragePathOptions& opts, |
131 | | int64_t expiration_secs, |
132 | | const S3ClientConf& conf) = 0; |
133 | | }; |
134 | | } // namespace io |
135 | | } // namespace doris |