common/cpp/obj-client/s3_obj_storage_client.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "s3_obj_storage_client.h" |
19 | | |
20 | | #include <cpp/obj-client/obj_storage_client.h> |
21 | | #include <gen_cpp/Status_types.h> |
22 | | |
23 | | #include <algorithm> |
24 | | #include <chrono> |
25 | | |
26 | | #include "client_bvar.h" |
27 | | #include "cpp/obj_retry_strategy.h" |
28 | | |
29 | | namespace Aws::S3::Model { |
30 | | class DeleteObjectRequest; |
31 | | } // namespace Aws::S3::Model |
32 | | |
33 | | using Aws::S3::Model::CompletedPart; |
34 | | using Aws::S3::Model::CompletedMultipartUpload; |
35 | | using Aws::S3::Model::CompleteMultipartUploadRequest; |
36 | | using Aws::S3::Model::CreateMultipartUploadRequest; |
37 | | using Aws::S3::Model::UploadPartRequest; |
38 | | using Aws::S3::Model::UploadPartOutcome; |
39 | | |
40 | | namespace doris { |
41 | | using namespace Aws::S3::Model; |
42 | | namespace { |
43 | | |
44 | | constexpr int64_t S3_REQUEST_THRESHOLD_MS = 5000; |
45 | | |
46 | 22 | int64_t elapsed_time_milliseconds(std::chrono::steady_clock::time_point start) { |
47 | 22 | return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - |
48 | 22 | start) |
49 | 22 | .count(); |
50 | 22 | } |
51 | | |
52 | 6 | void record_s3_request_failed(const Aws::S3::S3Error& error) { |
53 | 6 | record_object_request_failed(static_cast<int>(error.GetResponseCode())); |
54 | 6 | } |
55 | | |
56 | 3 | std::string object_identity(const ObjStoragePath& opts) { |
57 | 3 | return opts.path.empty() ? opts.key : opts.path.native(); |
58 | 3 | } |
59 | | |
60 | 10 | std::string s3_error_message(const Aws::S3::S3Error& error, std::string_view message) { |
61 | 10 | return fmt::format("{}: {} {} code={}, type={}, request_id={}", message, |
62 | 10 | error.GetExceptionName(), error.GetMessage(), |
63 | 10 | static_cast<int>(error.GetResponseCode()), |
64 | 10 | static_cast<int>(error.GetErrorType()), error.GetRequestId()); |
65 | 10 | } |
66 | | |
67 | | } // namespace |
68 | | |
69 | 10 | ObjStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) { |
70 | 10 | return obj_storage_status_from_http_code(static_cast<int>(err.GetResponseCode()), |
71 | 10 | s3_error_message(err, msg)); |
72 | 10 | } |
73 | | |
74 | 8 | ObjStorageUploadResult S3ObjStorageClient::create_multipart_upload(const ObjStoragePath& opts) { |
75 | 8 | CreateMultipartUploadRequest request; |
76 | 8 | request.WithBucket(opts.bucket).WithKey(opts.key); |
77 | 8 | request.SetContentType("application/octet-stream"); |
78 | | |
79 | 8 | const auto start = std::chrono::steady_clock::now(); |
80 | 8 | auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( |
81 | 8 | [&]() { |
82 | 8 | client_bvar::ScopedLatency scoped_latency( |
83 | 8 | client_bvar::s3_multi_part_upload_latency); |
84 | 8 | return _client->CreateMultipartUpload(request); |
85 | 8 | }(), |
86 | 8 | "s3_file_writer::create_multi_part_upload", std::cref(request).get()); |
87 | 8 | SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); |
88 | 8 | const auto elapsed_ms = elapsed_time_milliseconds(start); |
89 | | |
90 | 8 | const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() |
91 | 8 | : outcome.GetError().GetRequestId(); |
92 | | |
93 | 8 | LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) |
94 | 0 | << "CreateMultipartUpload cost=" << elapsed_ms << "ms" |
95 | 0 | << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; |
96 | | |
97 | 8 | if (!outcome.IsSuccess()) { |
98 | 1 | record_s3_request_failed(outcome.GetError()); |
99 | 1 | auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", |
100 | 1 | opts.path.native())); |
101 | 1 | LOG(WARNING) << st.code << " request_id=" << request_id; |
102 | 1 | return ObjStorageUploadResult { |
103 | 1 | .resp = {.status = st, |
104 | 1 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
105 | 1 | .request_id = outcome.GetError().GetRequestId()}, |
106 | 1 | }; |
107 | 1 | } |
108 | | |
109 | 7 | return ObjStorageUploadResult {.resp = ObjStorageResponse::OK(), |
110 | 7 | .upload_id {outcome.GetResult().GetUploadId()}}; |
111 | 8 | } |
112 | | |
113 | | ObjStorageResponse S3ObjStorageClient::put_object(const ObjStoragePath& opts, |
114 | 1 | std::string_view stream) { |
115 | 1 | Aws::S3::Model::PutObjectRequest request; |
116 | 1 | request.WithBucket(opts.bucket).WithKey(opts.key); |
117 | 1 | auto string_view_stream = std::make_shared<StringViewStream>(stream.data(), stream.size()); |
118 | 1 | Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); |
119 | 1 | request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); |
120 | 1 | request.SetBody(string_view_stream); |
121 | 1 | request.SetContentLength(stream.size()); |
122 | 1 | request.SetContentType("application/octet-stream"); |
123 | | |
124 | 1 | const auto start = std::chrono::steady_clock::now(); |
125 | 1 | auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( |
126 | 1 | [&]() { |
127 | 1 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_put_latency); |
128 | 1 | return _client->PutObject(request); |
129 | 1 | }(), |
130 | 1 | "s3_file_writer::put_object", std::cref(request).get(), &stream); |
131 | 1 | const auto elapsed_ms = elapsed_time_milliseconds(start); |
132 | | |
133 | 1 | const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() |
134 | 1 | : outcome.GetError().GetRequestId(); |
135 | | |
136 | 1 | if (!outcome.IsSuccess()) { |
137 | 0 | record_s3_request_failed(outcome.GetError()); |
138 | 0 | auto st = s3fs_error(outcome.GetError(), |
139 | 0 | fmt::format("failed to put object: {}", opts.path.native())); |
140 | 0 | LOG(WARNING) << st.code << ", request_id=" << request_id; |
141 | 0 | return ObjStorageResponse { |
142 | 0 | .status = st, |
143 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
144 | 0 | .request_id = outcome.GetError().GetRequestId()}; |
145 | 0 | } |
146 | | |
147 | 1 | LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) |
148 | 0 | << "PutObject cost=" << elapsed_ms << "ms" |
149 | 0 | << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; |
150 | 1 | return ObjStorageResponse::OK(); |
151 | 1 | } |
152 | | |
153 | | ObjStorageUploadResult S3ObjStorageClient::upload_part(const ObjStoragePath& opts, |
154 | | const std::string& upload_id, |
155 | 11 | std::string_view stream, int part_num) { |
156 | 11 | UploadPartRequest request; |
157 | 11 | request.WithBucket(opts.bucket) |
158 | 11 | .WithKey(opts.key) |
159 | 11 | .WithPartNumber(part_num) |
160 | 11 | .WithUploadId(upload_id); |
161 | 11 | auto string_view_stream = std::make_shared<StringViewStream>(stream.data(), stream.size()); |
162 | | |
163 | 11 | request.SetBody(string_view_stream); |
164 | | |
165 | 11 | Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); |
166 | 11 | request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); |
167 | | |
168 | 11 | request.SetContentLength(stream.size()); |
169 | 11 | request.SetContentType("application/octet-stream"); |
170 | | |
171 | 11 | const auto start = std::chrono::steady_clock::now(); |
172 | 11 | auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( |
173 | 11 | [&]() { |
174 | 11 | client_bvar::ScopedLatency scoped_latency( |
175 | 11 | client_bvar::s3_multi_part_upload_latency); |
176 | | |
177 | 11 | return _client->UploadPart(request); |
178 | 11 | }(), |
179 | 11 | "s3_file_writer::upload_part", std::cref(request).get(), &stream); |
180 | 11 | const auto elapsed_ms = elapsed_time_milliseconds(start); |
181 | | |
182 | 11 | const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() |
183 | 11 | : outcome.GetError().GetRequestId(); |
184 | | |
185 | 11 | TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); |
186 | 11 | if (!outcome.IsSuccess()) { |
187 | 1 | record_s3_request_failed(outcome.GetError()); |
188 | 1 | auto st = s3fs_error(outcome.GetError(), |
189 | 1 | fmt::format("failed to UploadPart: {}, part_num {}, upload_id={}", |
190 | 1 | opts.path.native(), part_num, upload_id)); |
191 | | |
192 | 1 | LOG(WARNING) << st.code << ", request_id=" << request_id; |
193 | 1 | return ObjStorageUploadResult { |
194 | 1 | .resp = {.status = st, |
195 | 1 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
196 | 1 | .request_id = outcome.GetError().GetRequestId()}}; |
197 | 1 | } |
198 | 11 | LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) |
199 | 0 | << "UploadPart cost=" << elapsed_ms << "ms" |
200 | 0 | << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key |
201 | 0 | << ", part_num=" << part_num << ", upload_id=" << upload_id; |
202 | 10 | return ObjStorageUploadResult {.resp = ObjStorageResponse::OK(), |
203 | 10 | .etag = outcome.GetResult().GetETag()}; |
204 | 11 | } |
205 | | |
206 | | ObjStorageResponse S3ObjStorageClient::complete_multipart_upload( |
207 | | const ObjStoragePath& opts, const std::string& upload_id, |
208 | 3 | const std::vector<ObjStorageCompletedPart>& completed_parts) { |
209 | 3 | CompleteMultipartUploadRequest request; |
210 | 3 | request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(upload_id); |
211 | | |
212 | 3 | CompletedMultipartUpload completed_upload; |
213 | 3 | std::vector<CompletedPart> complete_parts; |
214 | 3 | std::ranges::transform(completed_parts, std::back_inserter(complete_parts), |
215 | 6 | [](const ObjStorageCompletedPart& part_ptr) { |
216 | 6 | CompletedPart part; |
217 | 6 | part.SetPartNumber(part_ptr.part_num); |
218 | 6 | part.SetETag(part_ptr.etag); |
219 | 6 | return part; |
220 | 6 | }); |
221 | 3 | completed_upload.SetParts(std::move(complete_parts)); |
222 | 3 | request.WithMultipartUpload(completed_upload); |
223 | | |
224 | 3 | TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjStorageResponse(), this); |
225 | | |
226 | 2 | const auto start = std::chrono::steady_clock::now(); |
227 | 2 | auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( |
228 | 2 | [&]() { |
229 | 2 | client_bvar::ScopedLatency scoped_latency( |
230 | 2 | client_bvar::s3_multi_part_upload_latency); |
231 | 2 | return _client->CompleteMultipartUpload(request); |
232 | 2 | }(), |
233 | 2 | "s3_file_writer::complete_multi_part", std::cref(request).get()); |
234 | 2 | const auto elapsed_ms = elapsed_time_milliseconds(start); |
235 | | |
236 | 2 | const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() |
237 | 2 | : outcome.GetError().GetRequestId(); |
238 | | |
239 | 2 | if (!outcome.IsSuccess()) { |
240 | 1 | record_s3_request_failed(outcome.GetError()); |
241 | 1 | auto st = s3fs_error(outcome.GetError(), |
242 | 1 | fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", |
243 | 1 | opts.path.native(), upload_id)); |
244 | 1 | LOG(WARNING) << st.code << ", request_id=" << request_id; |
245 | 1 | return {.status = st, |
246 | 1 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
247 | 1 | .request_id = outcome.GetError().GetRequestId()}; |
248 | 1 | } |
249 | | |
250 | 2 | LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) |
251 | 0 | << "CompleteMultipartUpload cost=" << elapsed_ms << "ms" |
252 | 0 | << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key |
253 | 0 | << ", upload_id=" << upload_id; |
254 | 1 | return ObjStorageResponse::OK(); |
255 | 2 | } |
256 | | |
257 | 15 | ObjStorageHeadResult S3ObjStorageClient::head_object(const ObjStoragePath& opts) { |
258 | 15 | Aws::S3::Model::HeadObjectRequest request; |
259 | 15 | request.WithBucket(opts.bucket).WithKey(opts.key); |
260 | | |
261 | 15 | auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( |
262 | 15 | [&]() { |
263 | 15 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); |
264 | 15 | return _client->HeadObject(request); |
265 | 15 | }(), |
266 | 15 | "s3_file_system::head_object", std::ref(request).get()); |
267 | | |
268 | 15 | if (outcome.IsSuccess()) { |
269 | 8 | return {.resp = ObjStorageResponse::OK(), |
270 | 8 | .file_size = outcome.GetResult().GetContentLength()}; |
271 | 8 | } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { |
272 | 4 | return {.resp = {.status = ObjStorageStatus::NOT_FOUND}, .file_size = 0}; |
273 | 4 | } else { |
274 | 3 | record_s3_request_failed(outcome.GetError()); |
275 | 3 | LOG(WARNING) << "failed to head object" |
276 | 3 | << "bucket " << opts.bucket << " key " << opts.key << " responseCode " |
277 | 3 | << outcome.GetError() << " error " << outcome.GetError().GetMessage() |
278 | 3 | << " request_id " << outcome.GetError().GetRequestId(); |
279 | 3 | return {.resp = {.status = s3fs_error( |
280 | 3 | outcome.GetError(), |
281 | 3 | fmt::format("failed to head object: {}", object_identity(opts))), |
282 | 3 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
283 | 3 | .request_id = outcome.GetError().GetRequestId()}, |
284 | 3 | .file_size = -1}; |
285 | 3 | } |
286 | 15 | } |
287 | | |
288 | | ObjStorageResponse S3ObjStorageClient::get_object(const ObjStoragePath& opts, void* buffer, |
289 | | size_t offset, size_t bytes_read, |
290 | 0 | size_t* size_return) { |
291 | 0 | Aws::S3::Model::GetObjectRequest request; |
292 | 0 | request.WithBucket(opts.bucket).WithKey(opts.key); |
293 | 0 | request.SetRange(fmt::format("bytes={}-{}", offset, offset + bytes_read - 1)); |
294 | 0 | request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); |
295 | |
|
296 | 0 | auto outcome = [&]() { |
297 | 0 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_get_latency); |
298 | 0 | return _client->GetObject(request); |
299 | 0 | }(); |
300 | 0 | if (!outcome.IsSuccess()) { |
301 | 0 | record_s3_request_failed(outcome.GetError()); |
302 | 0 | return ObjStorageResponse { |
303 | 0 | .status = s3fs_error(outcome.GetError(), fmt::format("failed to get object: {}", |
304 | 0 | object_identity(opts))), |
305 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
306 | 0 | .request_id = outcome.GetError().GetRequestId(), |
307 | 0 | }; |
308 | 0 | } |
309 | 0 | *size_return = outcome.GetResult().GetContentLength(); |
310 | 0 | SYNC_POINT_CALLBACK("s3_obj_storage_client::get_object", size_return); |
311 | 0 | if (*size_return != bytes_read) { |
312 | 0 | const auto& request_id = outcome.GetResult().GetRequestId(); |
313 | 0 | return ObjStorageResponse { |
314 | 0 | .status = {ObjStorageStatus::INTERNAL_ERROR, |
315 | 0 | fmt::format("incomplete read from {}, expect {}, got {}, request_id={}", |
316 | 0 | object_identity(opts), bytes_read, *size_return, |
317 | 0 | request_id)}, |
318 | 0 | .request_id = request_id}; |
319 | 0 | } |
320 | 0 | return ObjStorageResponse::OK(); |
321 | 0 | } |
322 | | |
323 | | ObjStorageListPageResult S3ObjStorageClient::list_objects_page( |
324 | 4 | const ObjStoragePath& opts, std::string_view continuation_token) { |
325 | 4 | const auto& prefix = opts.prefix.empty() ? opts.key : opts.prefix; |
326 | 4 | Aws::S3::Model::ListObjectsV2Request request; |
327 | 4 | request.WithBucket(opts.bucket) |
328 | 4 | .WithPrefix(prefix) |
329 | 4 | .WithMaxKeys(static_cast<int>(capabilities().max_list_page)); |
330 | 4 | if (!continuation_token.empty()) { |
331 | 2 | request.SetContinuationToken(std::string(continuation_token)); |
332 | 2 | } |
333 | 4 | TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::list_objects", &request); |
334 | | |
335 | 4 | auto outcome = [&]() { |
336 | 4 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); |
337 | 4 | return _client->ListObjectsV2(request); |
338 | 4 | }(); |
339 | | |
340 | 4 | const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() |
341 | 4 | : outcome.GetError().GetRequestId(); |
342 | 4 | if (!outcome.IsSuccess()) { |
343 | | // Some S3-compatible providers (for example TOS) return NoSuchKey instead of an empty page |
344 | | // when a prefix does not exist. |
345 | 0 | if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { |
346 | 0 | LOG(INFO) << fmt::format( |
347 | 0 | "NoSuchKey when listing objects, treat as empty response, endpoint: {}, " |
348 | 0 | "bucket: {}, prefix: {}, request_id: {}", |
349 | 0 | _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); |
350 | 0 | return {.resp = ObjStorageResponse::OK()}; |
351 | 0 | } |
352 | 0 | record_object_request_failed(static_cast<int>(outcome.GetError().GetResponseCode())); |
353 | 0 | const auto status = s3fs_error(outcome.GetError(), |
354 | 0 | fmt::format("failed to list objects: {}, prefix: {}", |
355 | 0 | request.GetBucket(), request.GetPrefix())); |
356 | 0 | LOG(WARNING) << fmt::format( |
357 | 0 | "failed to list objects, endpoint: {}, bucket: {}, prefix: {}, responseCode: {}, " |
358 | 0 | "error: {}, request_id: {}", |
359 | 0 | _config.endpoint, request.GetBucket(), request.GetPrefix(), |
360 | 0 | static_cast<int>(outcome.GetError().GetResponseCode()), |
361 | 0 | outcome.GetError().GetMessage(), request_id); |
362 | 0 | return { |
363 | 0 | .resp = {.status = status, |
364 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
365 | 0 | .request_id = request_id}, |
366 | 0 | }; |
367 | 0 | } |
368 | | |
369 | 4 | const auto& result = outcome.GetResult(); |
370 | 4 | if (result.GetIsTruncated() && result.GetNextContinuationToken().empty()) { |
371 | 1 | LOG(WARNING) << fmt::format( |
372 | 1 | "failed to list objects, isTruncated but no continuation token, endpoint: {}, " |
373 | 1 | "bucket: {}, prefix: {}, request_id: {}", |
374 | 1 | _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); |
375 | 1 | return { |
376 | 1 | .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, |
377 | 1 | fmt::format("failed to list objects: {}, prefix: {}", |
378 | 1 | request.GetBucket(), request.GetPrefix())}, |
379 | 1 | .http_code = 0, |
380 | 1 | .request_id = request_id}, |
381 | 1 | }; |
382 | 1 | } |
383 | | |
384 | 3 | ObjStorageListPageResult page { |
385 | 3 | .resp = ObjStorageResponse::OK(), |
386 | 3 | .continuation_token = result.GetNextContinuationToken(), |
387 | 3 | .has_more = result.GetIsTruncated(), |
388 | 3 | }; |
389 | 3 | const auto& content = result.GetContents(); |
390 | 3 | page.objects.reserve(content.size()); |
391 | 5 | for (const auto& obj : content) { |
392 | 5 | DCHECK(obj.GetKey().starts_with(request.GetPrefix())) |
393 | 0 | << obj.GetKey() << ' ' << request.GetPrefix(); |
394 | 5 | page.objects.emplace_back(ObjectMeta {.key = obj.GetKey(), |
395 | 5 | .size = obj.GetSize(), |
396 | 5 | .mtime_s = obj.GetLastModified().Seconds()}); |
397 | 5 | } |
398 | 3 | return page; |
399 | 4 | } |
400 | | |
401 | | ObjStorageResponse S3ObjStorageClient::delete_objects(const ObjStoragePath& opts, |
402 | 1 | std::vector<std::string> objs) { |
403 | 1 | size_t max_delete_batch = 1000; |
404 | 1 | TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_objects", &max_delete_batch); |
405 | 1 | TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_objects", &max_delete_batch); |
406 | 1 | max_delete_batch = std::max<size_t>(1, max_delete_batch); |
407 | 2 | for (size_t begin = 0; begin < objs.size(); begin += max_delete_batch) { |
408 | 1 | const size_t end = std::min(begin + max_delete_batch, objs.size()); |
409 | 1 | Aws::S3::Model::DeleteObjectsRequest delete_request; |
410 | 1 | delete_request.SetBucket(opts.bucket); |
411 | 1 | Aws::S3::Model::Delete del; |
412 | 1 | Aws::Vector<Aws::S3::Model::ObjectIdentifier> objects; |
413 | 1 | objects.reserve(end - begin); |
414 | 2 | for (size_t i = begin; i < end; ++i) { |
415 | 1 | Aws::S3::Model::ObjectIdentifier object; |
416 | 1 | object.SetKey(std::move(objs[i])); |
417 | 1 | objects.emplace_back(std::move(object)); |
418 | 1 | } |
419 | 1 | del.WithObjects(std::move(objects)).SetQuiet(true); |
420 | 1 | delete_request.SetDelete(std::move(del)); |
421 | | |
422 | 1 | auto delete_outcome = [&]() { |
423 | 1 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_objects_latency); |
424 | 1 | return _client->DeleteObjects(delete_request); |
425 | 1 | }(); |
426 | 1 | SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); |
427 | 1 | SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", &delete_outcome); |
428 | 1 | if (!delete_outcome.IsSuccess()) { |
429 | 0 | record_s3_request_failed(delete_outcome.GetError()); |
430 | 0 | LOG(WARNING) << fmt::format( |
431 | 0 | "failed to delete objects, endpoint: {}, bucket: {}, key: {}, responseCode: " |
432 | 0 | "{}, error: {}, request_id: {}", |
433 | 0 | _config.endpoint, opts.bucket, |
434 | 0 | delete_request.GetDelete().GetObjects().front().GetKey(), |
435 | 0 | static_cast<int>(delete_outcome.GetError().GetResponseCode()), |
436 | 0 | delete_outcome.GetError().GetMessage(), |
437 | 0 | delete_outcome.GetError().GetRequestId()); |
438 | 0 | return ObjStorageResponse { |
439 | 0 | .status = s3fs_error(delete_outcome.GetError(), |
440 | 0 | fmt::format("failed to delete dir {}", opts.key)), |
441 | 0 | .http_code = static_cast<int>(delete_outcome.GetError().GetResponseCode()), |
442 | 0 | .request_id = delete_outcome.GetError().GetRequestId()}; |
443 | 0 | } |
444 | 1 | if (!delete_outcome.GetResult().GetErrors().empty()) { |
445 | 0 | const auto& error = delete_outcome.GetResult().GetErrors().front(); |
446 | 0 | LOG(WARNING) << fmt::format( |
447 | 0 | "failed to delete object in batch, endpoint: {}, bucket: {}, key: {}, error " |
448 | 0 | "code: {}, error: {}, request_id: {}", |
449 | 0 | _config.endpoint, opts.bucket, error.GetKey(), error.GetCode(), |
450 | 0 | error.GetMessage(), delete_outcome.GetResult().GetRequestId()); |
451 | 0 | return ObjStorageResponse { |
452 | 0 | .status = {ObjStorageStatus::INTERNAL_ERROR, |
453 | 0 | fmt::format("failed to delete object {}: {}, request_id={}", |
454 | 0 | error.GetKey(), error.GetMessage(), |
455 | 0 | delete_outcome.GetResult().GetRequestId())}, |
456 | 0 | .request_id = delete_outcome.GetResult().GetRequestId()}; |
457 | 0 | } |
458 | 1 | } |
459 | 1 | return ObjStorageResponse::OK(); |
460 | 1 | } |
461 | | |
462 | 1 | ObjStorageResponse S3ObjStorageClient::delete_object(const ObjStoragePath& opts) { |
463 | 1 | Aws::S3::Model::DeleteObjectRequest request; |
464 | 1 | request.WithBucket(opts.bucket).WithKey(opts.key); |
465 | | |
466 | 1 | auto outcome = [&]() { |
467 | 1 | client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_object_latency); |
468 | | |
469 | 1 | return _client->DeleteObject(request); |
470 | 1 | }(); |
471 | 1 | TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); |
472 | 1 | TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_object", &outcome); |
473 | 1 | if (outcome.IsSuccess()) { |
474 | 0 | return ObjStorageResponse::OK(); |
475 | 0 | } |
476 | 1 | ObjStorageResponse response { |
477 | 1 | .status = s3fs_error(outcome.GetError(), |
478 | 1 | fmt::format("failed to delete object {}", opts.key)), |
479 | 1 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
480 | 1 | .request_id = outcome.GetError().GetRequestId()}; |
481 | 1 | if (response.status.code == ObjStorageStatus::NOT_FOUND) { |
482 | 1 | return response; |
483 | 1 | } |
484 | 0 | record_s3_request_failed(outcome.GetError()); |
485 | 0 | LOG(WARNING) << fmt::format( |
486 | 0 | "failed to delete object, endpoint: {}, bucket: {}, key: {}, responseCode: {}, " |
487 | 0 | "error: {}, request_id: {}", |
488 | 0 | _config.endpoint, opts.bucket, opts.key, |
489 | 0 | static_cast<int>(outcome.GetError().GetResponseCode()), outcome.GetError().GetMessage(), |
490 | 0 | outcome.GetError().GetRequestId()); |
491 | 0 | return response; |
492 | 1 | } |
493 | | |
494 | | std::string S3ObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, |
495 | 0 | int64_t expiration_secs) { |
496 | 0 | return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, |
497 | 0 | expiration_secs); |
498 | 0 | } |
499 | | |
500 | 0 | ObjStorageResponse S3ObjStorageClient::check_versioning(const std::string& bucket) { |
501 | 0 | Aws::S3::Model::GetBucketVersioningRequest request; |
502 | 0 | request.SetBucket(bucket); |
503 | |
|
504 | 0 | auto outcome = _client->GetBucketVersioning(request); |
505 | |
|
506 | 0 | if (outcome.IsSuccess()) { |
507 | 0 | const auto& versioning_configuration = outcome.GetResult().GetStatus(); |
508 | 0 | if (versioning_configuration != Aws::S3::Model::BucketVersioningStatus::Enabled) { |
509 | 0 | LOG(WARNING) << "Err for check interval: bucket doesn't enable bucket versioning" |
510 | 0 | << " endpoint=" << _config.endpoint << " bucket=" << bucket; |
511 | 0 | return ObjStorageResponse { |
512 | 0 | .status = {ObjStorageStatus::INTERNAL_ERROR, |
513 | 0 | fmt::format("bucket versioning is not enabled: {}", bucket)}}; |
514 | 0 | } |
515 | 0 | } else { |
516 | 0 | record_s3_request_failed(outcome.GetError()); |
517 | 0 | LOG(WARNING) << "Err for check interval: failed to get status of bucket versioning" |
518 | 0 | << " endpoint=" << _config.endpoint << " bucket=" << bucket |
519 | 0 | << " responseCode=" << static_cast<int>(outcome.GetError().GetResponseCode()) |
520 | 0 | << " error=" << outcome.GetError().GetMessage() |
521 | 0 | << " request_id=" << outcome.GetError().GetRequestId(); |
522 | 0 | return ObjStorageResponse { |
523 | 0 | .status = s3fs_error(outcome.GetError(), |
524 | 0 | fmt::format("failed to get bucket versioning: {}", bucket)), |
525 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
526 | 0 | .request_id = outcome.GetError().GetRequestId()}; |
527 | 0 | } |
528 | 0 | return ObjStorageResponse::OK(); |
529 | 0 | } |
530 | | |
531 | | ObjStorageResponse S3ObjStorageClient::abort_multipart_upload(const ObjStoragePath& opts, |
532 | 0 | const std::string& upload_id) { |
533 | 0 | Aws::S3::Model::AbortMultipartUploadRequest request; |
534 | 0 | request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(upload_id); |
535 | |
|
536 | 0 | auto outcome = _client->AbortMultipartUpload(request); |
537 | 0 | if (!outcome.IsSuccess()) { |
538 | 0 | LOG(WARNING) << "failed to abort multipart upload" |
539 | 0 | << " endpoint=" << _config.endpoint << " bucket=" << opts.bucket |
540 | 0 | << " key=" << opts.key << " upload_id=" << upload_id |
541 | 0 | << " responseCode=" << static_cast<int>(outcome.GetError().GetResponseCode()) |
542 | 0 | << " error=" << outcome.GetError().GetMessage() |
543 | 0 | << " request_id=" << outcome.GetError().GetRequestId(); |
544 | 0 | if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { |
545 | 0 | return ObjStorageResponse::OK(); |
546 | 0 | } |
547 | 0 | record_s3_request_failed(outcome.GetError()); |
548 | 0 | return ObjStorageResponse { |
549 | 0 | .status = |
550 | 0 | s3fs_error(outcome.GetError(), |
551 | 0 | fmt::format("failed to abort multipart upload: {}, upload_id={}", |
552 | 0 | opts.path.native(), upload_id)), |
553 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
554 | 0 | .request_id = outcome.GetError().GetRequestId(), |
555 | 0 | }; |
556 | 0 | } |
557 | 0 | return ObjStorageResponse::OK(); |
558 | 0 | } |
559 | | |
560 | | ObjStorageResponse S3ObjStorageClient::get_lifecycle(const std::string& bucket, |
561 | 0 | int64_t* expiration_days) { |
562 | 0 | Aws::S3::Model::GetBucketLifecycleConfigurationRequest request; |
563 | 0 | request.SetBucket(bucket); |
564 | |
|
565 | 0 | auto outcome = _client->GetBucketLifecycleConfiguration(request); |
566 | 0 | bool has_lifecycle = false; |
567 | 0 | if (outcome.IsSuccess()) { |
568 | 0 | const auto& rules = outcome.GetResult().GetRules(); |
569 | 0 | for (const auto& rule : rules) { |
570 | 0 | if (rule.NoncurrentVersionExpirationHasBeenSet()) { |
571 | 0 | has_lifecycle = true; |
572 | 0 | *expiration_days = rule.GetNoncurrentVersionExpiration().GetNoncurrentDays(); |
573 | 0 | } |
574 | 0 | } |
575 | 0 | } else { |
576 | 0 | record_s3_request_failed(outcome.GetError()); |
577 | 0 | LOG(WARNING) << "Err for check interval: failed to get bucket lifecycle" |
578 | 0 | << " endpoint=" << _config.endpoint << " bucket=" << bucket |
579 | 0 | << " responseCode=" << static_cast<int>(outcome.GetError().GetResponseCode()) |
580 | 0 | << " error=" << outcome.GetError().GetMessage() |
581 | 0 | << " request_id=" << outcome.GetError().GetRequestId(); |
582 | 0 | return ObjStorageResponse { |
583 | 0 | .status = s3fs_error(outcome.GetError(), |
584 | 0 | fmt::format("failed to get lift cycle: {}", bucket)), |
585 | 0 | .http_code = static_cast<int>(outcome.GetError().GetResponseCode()), |
586 | 0 | .request_id = outcome.GetError().GetRequestId()}; |
587 | 0 | } |
588 | | |
589 | 0 | if (!has_lifecycle) { |
590 | 0 | LOG(WARNING) << "Err for check interval: bucket doesn't have lifecycle configuration" |
591 | 0 | << " endpoint=" << _config.endpoint << " bucket=" << bucket; |
592 | 0 | return ObjStorageResponse { |
593 | 0 | .status = {ObjStorageStatus::NOT_FOUND, |
594 | 0 | fmt::format("bucket has no lifecycle configuration: {}", bucket)}}; |
595 | 0 | } |
596 | 0 | return ObjStorageResponse::OK(); |
597 | 0 | } |
598 | | |
599 | | } // namespace doris |