Coverage Report

Created: 2025-03-13 18:54

/root/doris/be/src/io/hdfs_util.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <bvar/bvar.h>
21
22
#include <atomic>
23
#include <cstdint>
24
#include <memory>
25
#include <string>
26
#include <unordered_map>
27
28
#include "common/kerberos/kerberos_ticket_cache.h"
29
#include "common/status.h"
30
#include "io/fs/hdfs.h"
31
#include "io/fs/path.h"
32
33
namespace cloud {
34
class HdfsVaultInfo;
35
}
36
37
namespace doris {
38
class HDFSCommonBuilder;
39
class THdfsParams;
40
41
namespace io {
42
43
class HdfsHandler {
44
public:
45
    hdfsFS hdfs_fs;
46
    bool is_kerberos_auth;
47
    std::string principal;
48
    std::string keytab_path;
49
    std::string fs_name;
50
    std::atomic<uint64_t> last_access_time;
51
    std::shared_ptr<kerberos::KerberosTicketCache> ticket_cache;
52
53
    HdfsHandler(hdfsFS fs, bool is_kerberos, const std::string& principal_,
54
                const std::string& keytab_path_, const std::string& fs_name_,
55
                std::shared_ptr<kerberos::KerberosTicketCache> ticket_cache_)
56
            : hdfs_fs(fs),
57
              is_kerberos_auth(is_kerberos),
58
              principal(principal_),
59
              keytab_path(keytab_path_),
60
              fs_name(fs_name_),
61
              last_access_time(std::time(nullptr)),
62
10
              ticket_cache(ticket_cache_) {}
63
64
10
    ~HdfsHandler() {
65
        // The ticket_cache will be automatically released when the last reference is gone
66
        // No need to explicitly cleanup kerberos ticket
67
10
    }
68
69
10
    void update_access_time() { last_access_time = std::time(nullptr); }
70
};
71
72
namespace hdfs_bvar {
73
extern bvar::LatencyRecorder hdfs_read_latency;
74
extern bvar::LatencyRecorder hdfs_write_latency;
75
extern bvar::LatencyRecorder hdfs_create_dir_latency;
76
extern bvar::LatencyRecorder hdfs_open_latency;
77
extern bvar::LatencyRecorder hdfs_close_latency;
78
extern bvar::LatencyRecorder hdfs_flush_latency;
79
extern bvar::LatencyRecorder hdfs_hflush_latency;
80
extern bvar::LatencyRecorder hdfs_hsync_latency;
81
}; // namespace hdfs_bvar
82
83
// if the format of path is hdfs://ip:port/path, replace it to /path.
84
// path like hdfs://ip:port/path can't be used by libhdfs3.
85
Path convert_path(const Path& path, const std::string& namenode);
86
87
std::string get_fs_name(const std::string& path);
88
89
// return true if path_or_fs contains "hdfs://"
90
bool is_hdfs(const std::string& path_or_fs);
91
92
THdfsParams to_hdfs_params(const cloud::HdfsVaultInfo& vault);
93
94
} // namespace io
95
} // namespace doris