Coverage Report

Created: 2024-11-20 19:28

/root/doris/be/src/util/s3_util.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <aws/core/Aws.h>
21
#include <aws/core/client/ClientConfiguration.h>
22
#include <bvar/bvar.h>
23
#include <fmt/format.h>
24
#include <stdint.h>
25
26
#include <map>
27
#include <memory>
28
#include <mutex>
29
#include <string>
30
#include <unordered_map>
31
32
#include "common/status.h"
33
#include "gutil/hash/hash.h"
34
35
namespace Aws {
36
namespace S3 {
37
class S3Client;
38
} // namespace S3
39
} // namespace Aws
40
namespace bvar {
41
template <typename T>
42
class Adder;
43
}
44
45
namespace doris {
46
47
namespace s3_bvar {
48
extern bvar::LatencyRecorder s3_get_latency;
49
extern bvar::LatencyRecorder s3_put_latency;
50
extern bvar::LatencyRecorder s3_delete_latency;
51
extern bvar::LatencyRecorder s3_head_latency;
52
extern bvar::LatencyRecorder s3_multi_part_upload_latency;
53
extern bvar::LatencyRecorder s3_list_latency;
54
extern bvar::LatencyRecorder s3_list_object_versions_latency;
55
extern bvar::LatencyRecorder s3_get_bucket_version_latency;
56
extern bvar::LatencyRecorder s3_copy_object_latency;
57
}; // namespace s3_bvar
58
59
class S3URI;
60
61
const static std::string S3_AK = "AWS_ACCESS_KEY";
62
const static std::string S3_SK = "AWS_SECRET_KEY";
63
const static std::string S3_ENDPOINT = "AWS_ENDPOINT";
64
const static std::string S3_REGION = "AWS_REGION";
65
const static std::string S3_TOKEN = "AWS_TOKEN";
66
const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE";
67
const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS";
68
const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS";
69
70
struct S3Conf {
71
    std::string ak;
72
    std::string sk;
73
    std::string token;
74
    std::string endpoint;
75
    std::string region;
76
    std::string bucket;
77
    std::string prefix;
78
    int max_connections = -1;
79
    int request_timeout_ms = -1;
80
    int connect_timeout_ms = -1;
81
    bool use_virtual_addressing = true;
82
83
0
    std::string to_string() const {
84
0
        return fmt::format(
85
0
                "(ak={}, sk=*, token={}, endpoint={}, region={}, bucket={}, prefix={}, "
86
0
                "max_connections={}, request_timeout_ms={}, connect_timeout_ms={}, "
87
0
                "use_virtual_addressing={})",
88
0
                ak, token, endpoint, region, bucket, prefix, max_connections, request_timeout_ms,
89
0
                connect_timeout_ms, use_virtual_addressing);
90
0
    }
91
92
1
    uint64_t get_hash() const {
93
1
        uint64_t hash_code = 0;
94
1
        hash_code += Fingerprint(ak);
95
1
        hash_code += Fingerprint(sk);
96
1
        hash_code += Fingerprint(token);
97
1
        hash_code += Fingerprint(endpoint);
98
1
        hash_code += Fingerprint(region);
99
1
        hash_code += Fingerprint(bucket);
100
1
        hash_code += Fingerprint(prefix);
101
1
        hash_code += Fingerprint(max_connections);
102
1
        hash_code += Fingerprint(request_timeout_ms);
103
1
        hash_code += Fingerprint(connect_timeout_ms);
104
1
        hash_code += Fingerprint(use_virtual_addressing);
105
1
        return hash_code;
106
1
    }
107
};
108
109
class S3ClientFactory {
110
public:
111
    ~S3ClientFactory();
112
113
    static S3ClientFactory& instance();
114
115
    std::shared_ptr<Aws::S3::S3Client> create(const S3Conf& s3_conf);
116
117
    static bool is_s3_conf_valid(const std::map<std::string, std::string>& prop);
118
119
    static bool is_s3_conf_valid(const S3Conf& s3_conf);
120
121
    static Status convert_properties_to_s3_conf(const std::map<std::string, std::string>& prop,
122
                                                const S3URI& s3_uri, S3Conf* s3_conf);
123
124
1
    static Aws::Client::ClientConfiguration& getClientConfiguration() {
125
        // The default constructor of ClientConfiguration will do some http call
126
        // such as Aws::Internal::GetEC2MetadataClient and other init operation,
127
        // which is unnecessary.
128
        // So here we use a static instance, and deep copy every time
129
        // to avoid unnecessary operations.
130
1
        static Aws::Client::ClientConfiguration instance;
131
1
        return instance;
132
1
    }
133
134
private:
135
    S3ClientFactory();
136
    static std::string get_valid_ca_cert_path();
137
138
    Aws::SDKOptions _aws_options;
139
    std::mutex _lock;
140
    std::unordered_map<uint64_t, std::shared_ptr<Aws::S3::S3Client>> _cache;
141
    std::string _ca_cert_file_path;
142
};
143
144
} // end namespace doris