/root/doris/be/src/util/s3_util.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <aws/core/Aws.h> |
21 | | #include <aws/core/client/ClientConfiguration.h> |
22 | | #include <bvar/bvar.h> |
23 | | #include <fmt/format.h> |
24 | | #include <stdint.h> |
25 | | |
26 | | #include <map> |
27 | | #include <memory> |
28 | | #include <mutex> |
29 | | #include <string> |
30 | | #include <unordered_map> |
31 | | |
32 | | #include "common/status.h" |
33 | | #include "gutil/hash/hash.h" |
34 | | |
35 | | namespace Aws { |
36 | | namespace S3 { |
37 | | class S3Client; |
38 | | } // namespace S3 |
39 | | } // namespace Aws |
40 | | namespace bvar { |
41 | | template <typename T> |
42 | | class Adder; |
43 | | } |
44 | | |
45 | | namespace doris { |
46 | | |
47 | | namespace s3_bvar { |
48 | | extern bvar::LatencyRecorder s3_get_latency; |
49 | | extern bvar::LatencyRecorder s3_put_latency; |
50 | | extern bvar::LatencyRecorder s3_delete_latency; |
51 | | extern bvar::LatencyRecorder s3_head_latency; |
52 | | extern bvar::LatencyRecorder s3_multi_part_upload_latency; |
53 | | extern bvar::LatencyRecorder s3_list_latency; |
54 | | extern bvar::LatencyRecorder s3_list_object_versions_latency; |
55 | | extern bvar::LatencyRecorder s3_get_bucket_version_latency; |
56 | | extern bvar::LatencyRecorder s3_copy_object_latency; |
57 | | }; // namespace s3_bvar |
58 | | |
59 | | class S3URI; |
60 | | |
61 | | const static std::string S3_AK = "AWS_ACCESS_KEY"; |
62 | | const static std::string S3_SK = "AWS_SECRET_KEY"; |
63 | | const static std::string S3_ENDPOINT = "AWS_ENDPOINT"; |
64 | | const static std::string S3_REGION = "AWS_REGION"; |
65 | | const static std::string S3_TOKEN = "AWS_TOKEN"; |
66 | | const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE"; |
67 | | const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS"; |
68 | | const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS"; |
69 | | |
70 | | struct S3Conf { |
71 | | std::string ak; |
72 | | std::string sk; |
73 | | std::string token; |
74 | | std::string endpoint; |
75 | | std::string region; |
76 | | std::string bucket; |
77 | | std::string prefix; |
78 | | int max_connections = -1; |
79 | | int request_timeout_ms = -1; |
80 | | int connect_timeout_ms = -1; |
81 | | bool use_virtual_addressing = true; |
82 | | |
83 | 0 | std::string to_string() const { |
84 | 0 | return fmt::format( |
85 | 0 | "(ak={}, sk=*, token={}, endpoint={}, region={}, bucket={}, prefix={}, " |
86 | 0 | "max_connections={}, request_timeout_ms={}, connect_timeout_ms={}, " |
87 | 0 | "use_virtual_addressing={})", |
88 | 0 | ak, token, endpoint, region, bucket, prefix, max_connections, request_timeout_ms, |
89 | 0 | connect_timeout_ms, use_virtual_addressing); |
90 | 0 | } |
91 | | |
92 | 1 | uint64_t get_hash() const { |
93 | 1 | uint64_t hash_code = 0; |
94 | 1 | hash_code += Fingerprint(ak); |
95 | 1 | hash_code += Fingerprint(sk); |
96 | 1 | hash_code += Fingerprint(token); |
97 | 1 | hash_code += Fingerprint(endpoint); |
98 | 1 | hash_code += Fingerprint(region); |
99 | 1 | hash_code += Fingerprint(bucket); |
100 | 1 | hash_code += Fingerprint(prefix); |
101 | 1 | hash_code += Fingerprint(max_connections); |
102 | 1 | hash_code += Fingerprint(request_timeout_ms); |
103 | 1 | hash_code += Fingerprint(connect_timeout_ms); |
104 | 1 | hash_code += Fingerprint(use_virtual_addressing); |
105 | 1 | return hash_code; |
106 | 1 | } |
107 | | }; |
108 | | |
109 | | class S3ClientFactory { |
110 | | public: |
111 | | ~S3ClientFactory(); |
112 | | |
113 | | static S3ClientFactory& instance(); |
114 | | |
115 | | std::shared_ptr<Aws::S3::S3Client> create(const S3Conf& s3_conf); |
116 | | |
117 | | static bool is_s3_conf_valid(const std::map<std::string, std::string>& prop); |
118 | | |
119 | | static bool is_s3_conf_valid(const S3Conf& s3_conf); |
120 | | |
121 | | static Status convert_properties_to_s3_conf(const std::map<std::string, std::string>& prop, |
122 | | const S3URI& s3_uri, S3Conf* s3_conf); |
123 | | |
124 | 1 | static Aws::Client::ClientConfiguration& getClientConfiguration() { |
125 | | // The default constructor of ClientConfiguration will do some http call |
126 | | // such as Aws::Internal::GetEC2MetadataClient and other init operation, |
127 | | // which is unnecessary. |
128 | | // So here we use a static instance, and deep copy every time |
129 | | // to avoid unnecessary operations. |
130 | 1 | static Aws::Client::ClientConfiguration instance; |
131 | 1 | return instance; |
132 | 1 | } |
133 | | |
134 | | private: |
135 | | S3ClientFactory(); |
136 | | static std::string get_valid_ca_cert_path(); |
137 | | |
138 | | Aws::SDKOptions _aws_options; |
139 | | std::mutex _lock; |
140 | | std::unordered_map<uint64_t, std::shared_ptr<Aws::S3::S3Client>> _cache; |
141 | | std::string _ca_cert_file_path; |
142 | | }; |
143 | | |
144 | | } // end namespace doris |