Coverage Report

Created: 2026-07-20 16:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/oss_util.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <bvar/bvar.h>
21
#include <fmt/format.h>
22
#include <gen_cpp/cloud.pb.h>
23
24
#include <memory>
25
#include <mutex>
26
#include <string>
27
28
#include "common/status.h"
29
#include "cpp/aws_common.h"
30
#include "cpp/oss_common.h"
31
#include "util/s3_util.h"
32
33
namespace AlibabaCloud {
34
namespace OSS {
35
class OssClient;
36
} // namespace OSS
37
} // namespace AlibabaCloud
38
39
namespace doris {
40
41
class ECSMetadataCredentialsProvider;
42
class OSSSTSCredentialProvider;
43
class OSSDefaultCredentialsProvider;
44
45
namespace oss_bvar {
46
extern bvar::LatencyRecorder oss_get_latency;
47
extern bvar::LatencyRecorder oss_put_latency;
48
extern bvar::LatencyRecorder oss_delete_object_latency;
49
extern bvar::LatencyRecorder oss_delete_objects_latency;
50
extern bvar::LatencyRecorder oss_head_latency;
51
extern bvar::LatencyRecorder oss_list_latency;
52
extern bvar::LatencyRecorder oss_multi_part_upload_latency;
53
} // namespace oss_bvar
54
55
struct OSSClientConf {
56
    std::string endpoint;
57
    std::string region;
58
    std::string ak;
59
    std::string sk;
60
    std::string token;
61
    std::string bucket;
62
    std::string role_arn;
63
    std::string external_id;
64
    std::string sts_endpoint;
65
66
    int max_connections = 100;
67
    int request_timeout_ms = 30000;
68
    int connect_timeout_ms = 10000;
69
70
    OSSCredProviderType cred_provider_type = OSSCredProviderType::DEFAULT;
71
72
0
    uint64_t get_hash() const {
73
0
        uint64_t hash_code = 0;
74
0
        hash_code ^= crc32_hash(ak + sk);
75
0
        hash_code ^= crc32_hash(token);
76
0
        hash_code ^= crc32_hash(endpoint);
77
0
        hash_code ^= crc32_hash(region);
78
0
        hash_code ^= crc32_hash(bucket);
79
0
        hash_code ^= crc32_hash(role_arn);
80
0
        hash_code ^= crc32_hash(external_id);
81
0
        hash_code ^= crc32_hash(sts_endpoint);
82
0
        hash_code ^= max_connections;
83
0
        hash_code ^= request_timeout_ms;
84
0
        hash_code ^= connect_timeout_ms;
85
0
        hash_code ^= static_cast<int>(cred_provider_type);
86
0
        return hash_code;
87
0
    }
88
89
0
    std::string to_string() const {
90
0
        return fmt::format(
91
0
                "(ak={}, endpoint={}, region={}, bucket={}, role_arn={}, "
92
0
                "max_connections={}, cred_provider_type={})",
93
0
                hide_access_key(ak), endpoint, region, bucket, role_arn, max_connections,
94
0
                static_cast<int>(cred_provider_type));
95
0
    }
96
};
97
98
struct OSSConf {
99
    std::string bucket;
100
    std::string prefix;
101
    OSSClientConf client_conf;
102
103
    static OSSConf get_oss_conf(const cloud::ObjectStoreInfoPB& obj_info);
104
105
0
    std::string to_string() const {
106
0
        return fmt::format("(bucket={}, prefix={}, client_conf={})", bucket, prefix,
107
0
                           client_conf.to_string());
108
0
    }
109
};
110
111
class OSSClientFactory {
112
public:
113
    ~OSSClientFactory();
114
115
    static OSSClientFactory& instance();
116
117
    std::shared_ptr<AlibabaCloud::OSS::OssClient> create(const OSSClientConf& oss_conf);
118
119
private:
120
    OSSClientFactory();
121
122
    std::mutex _lock;
123
    std::unordered_map<uint64_t, std::shared_ptr<AlibabaCloud::OSS::OssClient>> _cache;
124
    std::unordered_map<uint64_t, std::shared_ptr<ECSMetadataCredentialsProvider>>
125
            _ecs_credential_providers;
126
    std::unordered_map<uint64_t, std::shared_ptr<OSSSTSCredentialProvider>>
127
            _sts_credential_providers;
128
    std::unordered_map<uint64_t, std::shared_ptr<OSSDefaultCredentialsProvider>>
129
            _default_credential_providers;
130
    std::string _ca_cert_file_path;
131
};
132
133
} // namespace doris