Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <bvar/bvar.h> |
21 | | #include <fmt/format.h> |
22 | | #include <gen_cpp/cloud.pb.h> |
23 | | |
24 | | #include <memory> |
25 | | #include <mutex> |
26 | | #include <string> |
27 | | |
28 | | #include "common/status.h" |
29 | | #include "cpp/aws_common.h" |
30 | | #include "cpp/oss_common.h" |
31 | | #include "util/s3_util.h" |
32 | | |
33 | | namespace AlibabaCloud { |
34 | | namespace OSS { |
35 | | class OssClient; |
36 | | } // namespace OSS |
37 | | } // namespace AlibabaCloud |
38 | | |
39 | | namespace doris { |
40 | | |
41 | | class ECSMetadataCredentialsProvider; |
42 | | class OSSSTSCredentialProvider; |
43 | | class OSSDefaultCredentialsProvider; |
44 | | |
45 | | namespace oss_bvar { |
46 | | extern bvar::LatencyRecorder oss_get_latency; |
47 | | extern bvar::LatencyRecorder oss_put_latency; |
48 | | extern bvar::LatencyRecorder oss_delete_object_latency; |
49 | | extern bvar::LatencyRecorder oss_delete_objects_latency; |
50 | | extern bvar::LatencyRecorder oss_head_latency; |
51 | | extern bvar::LatencyRecorder oss_list_latency; |
52 | | extern bvar::LatencyRecorder oss_multi_part_upload_latency; |
53 | | } // namespace oss_bvar |
54 | | |
55 | | struct OSSClientConf { |
56 | | std::string endpoint; |
57 | | std::string region; |
58 | | std::string ak; |
59 | | std::string sk; |
60 | | std::string token; |
61 | | std::string bucket; |
62 | | std::string role_arn; |
63 | | std::string external_id; |
64 | | std::string sts_endpoint; |
65 | | |
66 | | int max_connections = 100; |
67 | | int request_timeout_ms = 30000; |
68 | | int connect_timeout_ms = 10000; |
69 | | |
70 | | OSSCredProviderType cred_provider_type = OSSCredProviderType::DEFAULT; |
71 | | |
72 | 0 | uint64_t get_hash() const { |
73 | 0 | uint64_t hash_code = 0; |
74 | 0 | hash_code ^= crc32_hash(ak + sk); |
75 | 0 | hash_code ^= crc32_hash(token); |
76 | 0 | hash_code ^= crc32_hash(endpoint); |
77 | 0 | hash_code ^= crc32_hash(region); |
78 | 0 | hash_code ^= crc32_hash(bucket); |
79 | 0 | hash_code ^= crc32_hash(role_arn); |
80 | 0 | hash_code ^= crc32_hash(external_id); |
81 | 0 | hash_code ^= crc32_hash(sts_endpoint); |
82 | 0 | hash_code ^= max_connections; |
83 | 0 | hash_code ^= request_timeout_ms; |
84 | 0 | hash_code ^= connect_timeout_ms; |
85 | 0 | hash_code ^= static_cast<int>(cred_provider_type); |
86 | 0 | return hash_code; |
87 | 0 | } |
88 | | |
89 | 0 | std::string to_string() const { |
90 | 0 | return fmt::format( |
91 | 0 | "(ak={}, endpoint={}, region={}, bucket={}, role_arn={}, " |
92 | 0 | "max_connections={}, cred_provider_type={})", |
93 | 0 | hide_access_key(ak), endpoint, region, bucket, role_arn, max_connections, |
94 | 0 | static_cast<int>(cred_provider_type)); |
95 | 0 | } |
96 | | }; |
97 | | |
98 | | struct OSSConf { |
99 | | std::string bucket; |
100 | | std::string prefix; |
101 | | OSSClientConf client_conf; |
102 | | |
103 | | static OSSConf get_oss_conf(const cloud::ObjectStoreInfoPB& obj_info); |
104 | | |
105 | 0 | std::string to_string() const { |
106 | 0 | return fmt::format("(bucket={}, prefix={}, client_conf={})", bucket, prefix, |
107 | 0 | client_conf.to_string()); |
108 | 0 | } |
109 | | }; |
110 | | |
111 | | class OSSClientFactory { |
112 | | public: |
113 | | ~OSSClientFactory(); |
114 | | |
115 | | static OSSClientFactory& instance(); |
116 | | |
117 | | std::shared_ptr<AlibabaCloud::OSS::OssClient> create(const OSSClientConf& oss_conf); |
118 | | |
119 | | private: |
120 | | OSSClientFactory(); |
121 | | |
122 | | std::mutex _lock; |
123 | | std::unordered_map<uint64_t, std::shared_ptr<AlibabaCloud::OSS::OssClient>> _cache; |
124 | | std::unordered_map<uint64_t, std::shared_ptr<ECSMetadataCredentialsProvider>> |
125 | | _ecs_credential_providers; |
126 | | std::unordered_map<uint64_t, std::shared_ptr<OSSSTSCredentialProvider>> |
127 | | _sts_credential_providers; |
128 | | std::unordered_map<uint64_t, std::shared_ptr<OSSDefaultCredentialsProvider>> |
129 | | _default_credential_providers; |
130 | | std::string _ca_cert_file_path; |
131 | | }; |
132 | | |
133 | | } // namespace doris |