Coverage Report

Created: 2024-11-21 14:31

/root/doris/be/src/util/hdfs_util.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/hdfs_util.h"
19
20
#include <ostream>
21
22
#include "common/logging.h"
23
#include "io/fs/err_utils.h"
24
#include "io/hdfs_builder.h"
25
26
namespace doris {
27
namespace io {
28
29
0
HDFSHandle& HDFSHandle::instance() {
30
0
    static HDFSHandle hdfs_handle;
31
0
    return hdfs_handle;
32
0
}
33
34
0
hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) {
35
0
    hdfsFS hdfs_fs = hdfsBuilderConnect(hdfs_builder.get());
36
0
    if (hdfs_fs == nullptr) {
37
0
        LOG(WARNING) << "connect to hdfs failed."
38
0
                     << ", error: " << hdfs_error();
39
0
        return nullptr;
40
0
    }
41
0
    return hdfs_fs;
42
0
}
43
44
0
Path convert_path(const Path& path, const std::string& namenode) {
45
0
    std::string fs_path;
46
0
    if (path.native().starts_with(namenode)) {
47
        // `path` is URI format, remove the namenode part in `path`
48
0
        fs_path = path.native().substr(namenode.size());
49
0
    } else {
50
0
        fs_path = path;
51
0
    }
52
53
    // Always use absolute path (start with '/') in hdfs
54
0
    if (fs_path.empty() || fs_path[0] != '/') {
55
0
        fs_path.insert(fs_path.begin(), '/');
56
0
    }
57
0
    return fs_path;
58
0
}
59
60
0
bool is_hdfs(const std::string& path_or_fs) {
61
0
    return path_or_fs.rfind("hdfs://") == 0;
62
0
}
63
64
} // namespace io
65
} // namespace doris