Coverage Report

Created: 2026-09-17 15:32

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/fs/file_writer.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <future>
21
#include <memory>
22
23
#include "common/status.h"
24
#include "io/cache/block_file_cache.h"
25
#include "io/cache/block_file_cache_factory.h"
26
#include "io/cache/file_cache_common.h"
27
#include "io/fs/file_reader_writer_fwd.h"
28
#include "io/fs/packed_slice_location.h"
29
#include "io/fs/path.h"
30
#include "util/slice.h"
31
32
namespace doris::io {
33
class FileSystem;
34
struct FileCacheAllocatorBuilder;
35
struct EncryptionInfo;
36
37
// Only affects remote file writers
38
struct FileWriterOptions {
39
    // S3 committer will start multipart uploading all files on BE side,
40
    // and then complete multipart upload these files on FE side.
41
    // If you do not complete multi parts of a file, the file will not be visible.
42
    // So in this way, the atomicity of a single file can be guaranteed. But it still cannot
43
    // guarantee the atomicity of multiple files.
44
    // Because hive committers have best-effort semantics,
45
    // this shortens the inconsistent time window.
46
    bool used_by_s3_committer = false;
47
    bool write_file_cache = false;
48
    bool allow_adaptive_file_cache_write = true;
49
    bool is_cold_data = false;
50
    bool sync_file_data = true;              // Whether flush data into storage system
51
    uint64_t file_cache_expiration_time = 0; // Relative time
52
    uint64_t approximate_bytes_to_write = 0; // Approximate bytes to write, used for file cache
53
};
54
55
struct AsyncCloseStatusPack {
56
    std::promise<Status> promise;
57
    std::future<Status> future;
58
};
59
60
class FileWriter {
61
public:
62
    enum class State : uint8_t {
63
        OPENED = 0,
64
        ASYNC_CLOSING,
65
        CLOSED,
66
    };
67
15.7k
    FileWriter() = default;
68
15.7k
    virtual ~FileWriter() = default;
69
70
    FileWriter(const FileWriter&) = delete;
71
    const FileWriter& operator=(const FileWriter&) = delete;
72
73
    // Normal close. Wait for all data to persist before returning.
74
    // If there is no data appended, an empty file will be persisted.
75
    virtual Status close(bool non_block = false) = 0;
76
77
    // Non-blocking probe for a previous close(true).
78
    // OK means close finished successfully. NeedSendAgain means close is still running.
79
    // Other errors mean close finished with error or the writer does not support this API.
80
    // NOTE: This method consumes the async close result when it is ready. The caller must
81
    // use it as the only completion path for that async close; mixing it with close(false)
82
    // or another try_finish_close consumer is not supported.
83
0
    virtual Status try_finish_close() {
84
0
        return Status::NotSupported("try_finish_close is not supported");
85
0
    }
86
87
23.1k
    Status append(const Slice& data) { return appendv(&data, 1); }
88
89
    virtual Status appendv(const Slice* data, size_t data_cnt) = 0;
90
91
    virtual const Path& path() const = 0;
92
93
    virtual size_t bytes_appended() const = 0;
94
95
    virtual State state() const = 0;
96
97
    // Gets the location of this file's slice in the packed file. Must be called after the writer
98
    // is closed. An empty packed_file_path means the file is not in a packed file. Writers that
99
    // wrap another writer must forward this call, so callers never need to downcast to
100
    // PackedFileWriter.
101
1
    virtual Status get_packed_slice_location(PackedSliceLocation* location) const {
102
1
        *location = PackedSliceLocation {};
103
1
        return Status::OK();
104
1
    }
105
106
9
    FileCacheAllocatorBuilder* cache_builder() const {
107
9
        return _cache_builder == nullptr ? nullptr : _cache_builder.get();
108
9
    }
109
110
protected:
111
2.09k
    void init_cache_builder(const FileWriterOptions* opts, const Path& path) {
112
2.09k
        if (!config::enable_file_cache || opts == nullptr) {
113
1.07k
            return;
114
1.07k
        }
115
116
1.02k
        io::UInt128Wrapper path_hash = BlockFileCache::hash(path.filename().native());
117
1.02k
        BlockFileCache* file_cache_ptr = FileCacheFactory::instance()->get_by_path(path_hash);
118
119
1.02k
        bool has_enough_file_cache_space = opts->allow_adaptive_file_cache_write &&
120
1.02k
                                           config::enable_file_cache_adaptive_write &&
121
1.02k
                                           (opts->approximate_bytes_to_write > 0) &&
122
1.02k
                                           (file_cache_ptr->approximate_available_cache_size() >
123
0
                                            opts->approximate_bytes_to_write);
124
125
1.02k
        VLOG_DEBUG << "path:" << path.filename().native()
126
1
                   << ", write_file_cache:" << opts->write_file_cache
127
1
                   << ", allow_adaptive_file_cache_write:" << opts->allow_adaptive_file_cache_write
128
1
                   << ", has_enough_file_cache_space:" << has_enough_file_cache_space
129
1
                   << ", approximate_bytes_to_write:" << opts->approximate_bytes_to_write
130
1
                   << ", file_cache_available_size:"
131
1
                   << file_cache_ptr->approximate_available_cache_size();
132
1.02k
        if (opts->write_file_cache || has_enough_file_cache_space) {
133
4
            _cache_builder = std::make_unique<FileCacheAllocatorBuilder>(FileCacheAllocatorBuilder {
134
4
                    opts ? opts->is_cold_data : false, opts ? opts->file_cache_expiration_time : 0,
135
4
                    path_hash, file_cache_ptr});
136
4
        }
137
1.02k
        return;
138
2.09k
    }
139
140
    std::unique_ptr<FileCacheAllocatorBuilder> _cache_builder =
141
            nullptr; // nullptr if disable write file cache
142
};
143
144
} // namespace doris::io