Coverage Report

Created: 2026-08-20 09:01

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
common/cpp/obj-client/azure_obj_storage_client.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <aws/core/utils/Array.h>
21
#include <aws/core/utils/HashingUtils.h>
22
#include <cpp/sync_point.h>
23
#include <fmt/core.h>
24
#include <gen_cpp/Status_types.h>
25
#include <glog/logging.h>
26
27
#include <algorithm>
28
#include <azure/core/http/http.hpp>
29
#include <azure/core/http/http_status_code.hpp>
30
#include <azure/core/io/body_stream.hpp>
31
#include <azure/storage/blobs.hpp>
32
#include <azure/storage/blobs/blob_batch.hpp>
33
#include <azure/storage/blobs/blob_client.hpp>
34
#ifdef USE_AZURE
35
#include <azure/core/diagnostics/logger.hpp>
36
#include <azure/storage/blobs/blob_container_client.hpp>
37
#endif
38
#include <azure/storage/blobs/blob_sas_builder.hpp>
39
#include <azure/storage/blobs/rest_client.hpp>
40
#include <azure/storage/common/account_sas_builder.hpp>
41
#include <azure/storage/common/storage_credential.hpp>
42
#include <azure/storage/common/storage_exception.hpp>
43
#include <chrono>
44
#include <exception>
45
#include <iterator>
46
#include <ranges>
47
48
#include "client_bvar.h"
49
#include "cpp/obj_retry_strategy.h"
50
#include "cpp/util.h"
51
#include "obj_storage_client.h"
52
#include "s3_common.h"
53
54
namespace Azure::Storage::Blobs {
55
class BlobContainerClient;
56
} // namespace Azure::Storage::Blobs
57
58
namespace doris {
59
60
using namespace Azure::Storage::Blobs;
61
62
static const Azure::DateTime SystemClockEpoch {1970, 1, 1};
63
64
bool is_azure_tls_ca_error_message(std::string_view message);
65
std::string build_azure_tls_debug_suffix(std::string_view error_message,
66
                                         std::string_view tls_debug_context);
67
std::string azure_multipart_block_id(std::string_view upload_id, int part_num);
68
std::string build_azure_batch_delete_failure_message(const ObjStoragePath& opts,
69
                                                     std::string_view key);
70
71
class ObjClientHolder;
72
73
class AzureObjStorageClient final : public ObjStorageClient {
74
public:
75
    AzureObjStorageClient(
76
            std::shared_ptr<Azure::Storage::Blobs::BlobContainerClient> client,
77
            ObjStorageEndpointInfo config,
78
            std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> credential = nullptr)
79
            : _config(std::move(config)),
80
              _client(std::move(client)),
81
0
              _credential(std::move(credential)) {}
82
0
    ~AzureObjStorageClient() override = default;
83
    ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override;
84
    ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override;
85
    ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id,
86
                                       std::string_view, int partNum) override;
87
    ObjStorageResponse complete_multipart_upload(
88
            const ObjStoragePath& opts, const std::string& upload_id,
89
            const std::vector<ObjStorageCompletedPart>& completed_parts) override;
90
    ObjStorageHeadResult head_object(const ObjStoragePath& opts) override;
91
    ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset,
92
                                  size_t bytes_read, size_t* size_return) override;
93
    ObjStorageResponse delete_objects(const ObjStoragePath& opts,
94
                                      std::vector<std::string> objs) override;
95
    ObjStorageResponse delete_object(const ObjStoragePath& opts) override;
96
    std::string generate_presigned_url(const ObjStoragePath& opts,
97
                                       int64_t expiration_secs) override;
98
    ObjStorageResponse get_lifecycle(const std::string& bucket, int64_t* expiration_days) override;
99
100
    ObjStorageResponse check_versioning(const std::string& bucket) override;
101
102
    ObjStorageResponse abort_multipart_upload(const ObjStoragePath& opts,
103
                                              const std::string& upload_id) override;
104
0
    ObjStorageCapabilities capabilities() const override {
105
0
        return {.max_delete_batch = 256, .max_list_page = 5000};
106
0
    }
107
108
protected:
109
    ObjStorageListPageResult list_objects_page(const ObjStoragePath& path,
110
                                               std::string_view continuation_token) override;
111
112
private:
113
    ObjStorageEndpointInfo _config;
114
    std::shared_ptr<Azure::Storage::Blobs::BlobContainerClient> _client;
115
    std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> _credential;
116
};
117
118
} // namespace doris
119
120
namespace doris::io {
121
using ::doris::AzureObjStorageClient;
122
using ::doris::azure_multipart_block_id;
123
using ::doris::build_azure_batch_delete_failure_message;
124
using ::doris::build_azure_tls_debug_suffix;
125
using ::doris::is_azure_tls_ca_error_message;
126
} // namespace doris::io