Coverage Report

Created: 2026-03-12 14:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/fs/http_file_reader.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <atomic>
21
#include <map>
22
#include <memory>
23
#include <string>
24
25
#include "common/status.h"
26
#include "io/fs/file_handle_cache.h"
27
#include "io/fs/file_reader.h"
28
#include "io/fs/file_system.h"
29
#include "runtime/runtime_profile.h"
30
#include "service/http/http_client.h"
31
#include "util/slice.h"
32
33
namespace doris::io {
34
typedef struct OpenFileInfo {
35
    Path path;
36
    std::map<std::string, std::string> extend_info;
37
} OpenFileInfo;
38
class HttpFileReader final : public FileReader {
39
public:
40
    static Result<FileReaderSPtr> create(const std::string& url,
41
                                         const std::map<std::string, std::string>& props,
42
                                         const FileReaderOptions& opts, RuntimeProfile* profile);
43
44
    explicit HttpFileReader(const OpenFileInfo& fileInfo, std::string url, int64_t mtime);
45
    ~HttpFileReader() override;
46
47
    Status open(const FileReaderOptions& opts);
48
    Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
49
                        const IOContext* io_ctx = nullptr) override;
50
    Status close() override;
51
0
    const Path& path() const override { return _path; }
52
0
    bool closed() const override { return _closed.load(std::memory_order_acquire); }
53
0
    size_t size() const override { return _file_size; }
54
55
0
    int64_t mtime() const override { return _mtime; }
56
57
private:
58
    // Prepare and initialize the HTTP client for a new request
59
    Status prepare_client(bool set_fail_on_error = true);
60
61
    // Detect if the HTTP server supports Range requests
62
    // Returns OK on success with _range_supported set appropriately
63
    Status detect_range_support();
64
65
    std::unique_ptr<char[]> _read_buffer;
66
    static constexpr size_t READ_BUFFER_SIZE = 1 << 20; // 1MB
67
    // Default maximum file size for servers that don't support Range requests
68
    static constexpr size_t DEFAULT_MAX_REQUEST_SIZE = 100 << 20; // 100MB
69
70
    size_t _buffer_start = 0;
71
    size_t _buffer_end = 0;
72
    bool _size_known = false;
73
    bool _range_supported = true;
74
    std::string _etag;
75
    bool _initialized = false;
76
    std::map<std::string, std::string> _extend_kv;
77
    size_t _file_size = static_cast<size_t>(-1);
78
    Path _path;
79
    std::string _url;
80
    int64_t _last_modified = 0;
81
    std::atomic<bool> _closed = false;
82
    std::unique_ptr<HttpClient> _client;
83
    int64_t _mtime;
84
85
    // Configuration for non-Range request handling
86
    bool _enable_range_request = true;                         // Whether Range request is required
87
    size_t _max_request_size_bytes = DEFAULT_MAX_REQUEST_SIZE; // Max size for non-Range downloads
88
89
    // Full file cache for non-Range mode to avoid repeated downloads
90
    std::string _full_file_cache;   // Cache complete file content
91
    bool _full_file_cached = false; // Whether full file has been cached
92
};
93
94
} // namespace doris::io