Coverage Report

Created: 2026-07-01 22:22

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/io_common.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/Types_types.h>
21
22
#include <set>
23
#include <string>
24
25
namespace doris {
26
27
enum class ReaderType : uint8_t {
28
    READER_QUERY = 0,
29
    READER_ALTER_TABLE = 1,
30
    READER_BASE_COMPACTION = 2,
31
    READER_CUMULATIVE_COMPACTION = 3,
32
    READER_CHECKSUM = 4,
33
    READER_COLD_DATA_COMPACTION = 5,
34
    READER_SEGMENT_COMPACTION = 6,
35
    READER_FULL_COMPACTION = 7,
36
    READER_BINLOG_COMPACTION = 8,
37
    READER_BINLOG = 9,
38
    UNKNOWN = 10
39
};
40
41
namespace io {
42
43
struct FileReaderStats {
44
    size_t read_calls = 0;
45
    size_t read_bytes = 0;
46
    int64_t read_time_ns = 0;
47
    size_t read_rows = 0;
48
};
49
50
struct FileCacheStatistics {
51
    int64_t num_local_io_total = 0;
52
    int64_t num_remote_io_total = 0;
53
    int64_t num_peer_io_total = 0;
54
    int64_t local_io_timer = 0;
55
    int64_t bytes_read_from_local = 0;
56
    int64_t bytes_read_from_remote = 0;
57
    int64_t bytes_read_from_peer = 0;
58
    int64_t remote_io_timer = 0;
59
    int64_t peer_io_timer = 0;
60
    int64_t remote_wait_timer = 0;
61
    int64_t write_cache_io_timer = 0;
62
    int64_t bytes_write_into_cache = 0;
63
    int64_t num_skip_cache_io_total = 0;
64
    int64_t read_cache_file_directly_timer = 0;
65
    int64_t cache_get_or_set_timer = 0;
66
    int64_t lock_wait_timer = 0;
67
    int64_t get_timer = 0;
68
    int64_t set_timer = 0;
69
70
    int64_t inverted_index_num_local_io_total = 0;
71
    int64_t inverted_index_num_remote_io_total = 0;
72
    int64_t inverted_index_num_peer_io_total = 0;
73
    int64_t inverted_index_bytes_read_from_local = 0;
74
    int64_t inverted_index_bytes_read_from_remote = 0;
75
    int64_t inverted_index_bytes_read_from_peer = 0;
76
    int64_t inverted_index_local_io_timer = 0;
77
    int64_t inverted_index_remote_io_timer = 0;
78
    int64_t inverted_index_peer_io_timer = 0;
79
    int64_t inverted_index_io_timer = 0;
80
81
    // Cross-CG / Same-CG peer read statistics
82
    int64_t num_cross_cg_peer_io_total = 0;
83
    int64_t bytes_read_from_cross_cg_peer = 0;
84
    int64_t cross_cg_peer_io_timer = 0; // nanoseconds
85
    int64_t num_same_cg_peer_io_total = 0;
86
    int64_t bytes_read_from_same_cg_peer = 0;
87
    int64_t same_cg_peer_io_timer = 0; // nanoseconds
88
    int64_t num_peer_race_peer_win = 0;
89
    int64_t num_peer_race_s3_win = 0;
90
    int64_t num_peer_lazy_fetch = 0;
91
    int64_t peer_lazy_fetch_timer = 0; // nanoseconds
92
93
    std::set<std::string> peer_hosts;
94
95
0
    void merge_from(const FileCacheStatistics& other) {
96
0
        num_local_io_total += other.num_local_io_total;
97
0
        num_remote_io_total += other.num_remote_io_total;
98
0
        num_peer_io_total += other.num_peer_io_total;
99
0
        local_io_timer += other.local_io_timer;
100
0
        bytes_read_from_local += other.bytes_read_from_local;
101
0
        bytes_read_from_remote += other.bytes_read_from_remote;
102
0
        bytes_read_from_peer += other.bytes_read_from_peer;
103
0
        remote_io_timer += other.remote_io_timer;
104
0
        peer_io_timer += other.peer_io_timer;
105
0
        remote_wait_timer += other.remote_wait_timer;
106
0
        write_cache_io_timer += other.write_cache_io_timer;
107
0
        bytes_write_into_cache += other.bytes_write_into_cache;
108
0
        num_skip_cache_io_total += other.num_skip_cache_io_total;
109
0
        read_cache_file_directly_timer += other.read_cache_file_directly_timer;
110
0
        cache_get_or_set_timer += other.cache_get_or_set_timer;
111
0
        lock_wait_timer += other.lock_wait_timer;
112
0
        get_timer += other.get_timer;
113
0
        set_timer += other.set_timer;
114
115
0
        inverted_index_num_local_io_total += other.inverted_index_num_local_io_total;
116
0
        inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total;
117
0
        inverted_index_num_peer_io_total += other.inverted_index_num_peer_io_total;
118
0
        inverted_index_bytes_read_from_local += other.inverted_index_bytes_read_from_local;
119
0
        inverted_index_bytes_read_from_remote += other.inverted_index_bytes_read_from_remote;
120
0
        inverted_index_bytes_read_from_peer += other.inverted_index_bytes_read_from_peer;
121
0
        inverted_index_local_io_timer += other.inverted_index_local_io_timer;
122
0
        inverted_index_remote_io_timer += other.inverted_index_remote_io_timer;
123
0
        inverted_index_peer_io_timer += other.inverted_index_peer_io_timer;
124
0
        inverted_index_io_timer += other.inverted_index_io_timer;
125
126
0
        num_cross_cg_peer_io_total += other.num_cross_cg_peer_io_total;
127
0
        bytes_read_from_cross_cg_peer += other.bytes_read_from_cross_cg_peer;
128
0
        cross_cg_peer_io_timer += other.cross_cg_peer_io_timer;
129
0
        num_same_cg_peer_io_total += other.num_same_cg_peer_io_total;
130
0
        bytes_read_from_same_cg_peer += other.bytes_read_from_same_cg_peer;
131
0
        same_cg_peer_io_timer += other.same_cg_peer_io_timer;
132
0
        num_peer_race_peer_win += other.num_peer_race_peer_win;
133
0
        num_peer_race_s3_win += other.num_peer_race_s3_win;
134
0
        num_peer_lazy_fetch += other.num_peer_lazy_fetch;
135
0
        peer_lazy_fetch_timer += other.peer_lazy_fetch_timer;
136
137
0
        peer_hosts.insert(other.peer_hosts.begin(), other.peer_hosts.end());
138
0
    }
139
};
140
141
struct IOContext {
142
    ReaderType reader_type = ReaderType::UNKNOWN;
143
    // FIXME(plat1ko): Seems `is_disposable` can be inferred from the `reader_type`?
144
    bool is_disposable = false;
145
    bool is_index_data = false;
146
    bool read_file_cache = true;
147
    // TODO(lightman): use following member variables to control file cache
148
    bool is_persistent = false;
149
    // stop reader when reading, used in some interrupted operations
150
    bool should_stop = false;
151
    int64_t expiration_time = 0;
152
    const TUniqueId* query_id = nullptr;             // Ref
153
    FileCacheStatistics* file_cache_stats = nullptr; // Ref
154
    FileReaderStats* file_reader_stats = nullptr;    // Ref
155
    bool is_inverted_index = false;
156
    // if is_dryrun, read IO will download data to cache but return no data to reader
157
    // useful to skip cache data read from local disk to accelarate warm up
158
    bool is_dryrun = false;
159
    // if `is_warmup` == true, this I/O request is from a warm up task
160
    bool is_warmup {false};
161
    int64_t condition_cache_filtered_rows = 0;
162
    // if true, bypass peer read / peer-vs-S3 race and read directly from remote storage
163
    bool bypass_peer_read {false};
164
};
165
166
} // namespace io
167
} // namespace doris