Coverage Report

Created: 2026-07-03 18:10

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/io/io_common.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/Types_types.h>
21
22
#include <set>
23
#include <string>
24
25
namespace doris {
26
27
enum class ReaderType : uint8_t {
28
    READER_QUERY = 0,
29
    READER_ALTER_TABLE = 1,
30
    READER_BASE_COMPACTION = 2,
31
    READER_CUMULATIVE_COMPACTION = 3,
32
    READER_CHECKSUM = 4,
33
    READER_COLD_DATA_COMPACTION = 5,
34
    READER_SEGMENT_COMPACTION = 6,
35
    READER_FULL_COMPACTION = 7,
36
    READER_BINLOG_COMPACTION = 8,
37
    READER_BINLOG = 9,
38
    UNKNOWN = 10
39
};
40
41
namespace io {
42
43
struct FileReaderStats {
44
    size_t read_calls = 0;
45
    size_t read_bytes = 0;
46
    int64_t read_time_ns = 0;
47
    size_t read_rows = 0;
48
};
49
50
struct FileCacheStatistics {
51
    int64_t num_local_io_total = 0;
52
    int64_t num_remote_io_total = 0;
53
    int64_t num_peer_io_total = 0;
54
    int64_t local_io_timer = 0;
55
    int64_t bytes_read_from_local = 0;
56
    int64_t bytes_read_from_remote = 0;
57
    int64_t bytes_read_from_peer = 0;
58
    int64_t remote_io_timer = 0;
59
    int64_t peer_io_timer = 0;
60
    int64_t remote_wait_timer = 0;
61
    int64_t write_cache_io_timer = 0;
62
    int64_t bytes_write_into_cache = 0;
63
    int64_t num_skip_cache_io_total = 0;
64
    int64_t read_cache_file_directly_timer = 0;
65
    int64_t cache_get_or_set_timer = 0;
66
    int64_t lock_wait_timer = 0;
67
    int64_t get_timer = 0;
68
    int64_t set_timer = 0;
69
70
    int64_t inverted_index_num_local_io_total = 0;
71
    int64_t inverted_index_num_remote_io_total = 0;
72
    int64_t inverted_index_num_peer_io_total = 0;
73
    int64_t inverted_index_bytes_read_from_local = 0;
74
    int64_t inverted_index_bytes_read_from_remote = 0;
75
    int64_t inverted_index_bytes_read_from_peer = 0;
76
    int64_t inverted_index_local_io_timer = 0;
77
    int64_t inverted_index_remote_io_timer = 0;
78
    int64_t inverted_index_peer_io_timer = 0;
79
    int64_t inverted_index_io_timer = 0;
80
81
    int64_t segment_footer_index_num_local_io_total = 0;
82
    int64_t segment_footer_index_num_remote_io_total = 0;
83
    int64_t segment_footer_index_num_peer_io_total = 0;
84
    int64_t segment_footer_index_bytes_read_from_local = 0;
85
    int64_t segment_footer_index_bytes_read_from_remote = 0;
86
    int64_t segment_footer_index_bytes_read_from_peer = 0;
87
    int64_t segment_footer_index_local_io_timer = 0;
88
    int64_t segment_footer_index_remote_io_timer = 0;
89
    int64_t segment_footer_index_peer_io_timer = 0;
90
91
    // Cross-CG / Same-CG peer read statistics
92
    int64_t num_cross_cg_peer_io_total = 0;
93
    int64_t bytes_read_from_cross_cg_peer = 0;
94
    int64_t cross_cg_peer_io_timer = 0; // nanoseconds
95
    int64_t num_same_cg_peer_io_total = 0;
96
    int64_t bytes_read_from_same_cg_peer = 0;
97
    int64_t same_cg_peer_io_timer = 0; // nanoseconds
98
    int64_t num_peer_race_peer_win = 0;
99
    int64_t num_peer_race_s3_win = 0;
100
    int64_t num_peer_lazy_fetch = 0;
101
    int64_t peer_lazy_fetch_timer = 0; // nanoseconds
102
103
    std::set<std::string> peer_hosts;
104
105
0
    void merge_from(const FileCacheStatistics& other) {
106
0
        num_local_io_total += other.num_local_io_total;
107
0
        num_remote_io_total += other.num_remote_io_total;
108
0
        num_peer_io_total += other.num_peer_io_total;
109
0
        local_io_timer += other.local_io_timer;
110
0
        bytes_read_from_local += other.bytes_read_from_local;
111
0
        bytes_read_from_remote += other.bytes_read_from_remote;
112
0
        bytes_read_from_peer += other.bytes_read_from_peer;
113
0
        remote_io_timer += other.remote_io_timer;
114
0
        peer_io_timer += other.peer_io_timer;
115
0
        remote_wait_timer += other.remote_wait_timer;
116
0
        write_cache_io_timer += other.write_cache_io_timer;
117
0
        bytes_write_into_cache += other.bytes_write_into_cache;
118
0
        num_skip_cache_io_total += other.num_skip_cache_io_total;
119
0
        read_cache_file_directly_timer += other.read_cache_file_directly_timer;
120
0
        cache_get_or_set_timer += other.cache_get_or_set_timer;
121
0
        lock_wait_timer += other.lock_wait_timer;
122
0
        get_timer += other.get_timer;
123
0
        set_timer += other.set_timer;
124
125
0
        inverted_index_num_local_io_total += other.inverted_index_num_local_io_total;
126
0
        inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total;
127
0
        inverted_index_num_peer_io_total += other.inverted_index_num_peer_io_total;
128
0
        inverted_index_bytes_read_from_local += other.inverted_index_bytes_read_from_local;
129
0
        inverted_index_bytes_read_from_remote += other.inverted_index_bytes_read_from_remote;
130
0
        inverted_index_bytes_read_from_peer += other.inverted_index_bytes_read_from_peer;
131
0
        inverted_index_local_io_timer += other.inverted_index_local_io_timer;
132
0
        inverted_index_remote_io_timer += other.inverted_index_remote_io_timer;
133
0
        inverted_index_peer_io_timer += other.inverted_index_peer_io_timer;
134
0
        inverted_index_io_timer += other.inverted_index_io_timer;
135
136
0
        segment_footer_index_num_local_io_total += other.segment_footer_index_num_local_io_total;
137
0
        segment_footer_index_num_remote_io_total += other.segment_footer_index_num_remote_io_total;
138
0
        segment_footer_index_num_peer_io_total += other.segment_footer_index_num_peer_io_total;
139
0
        segment_footer_index_bytes_read_from_local +=
140
0
                other.segment_footer_index_bytes_read_from_local;
141
0
        segment_footer_index_bytes_read_from_remote +=
142
0
                other.segment_footer_index_bytes_read_from_remote;
143
0
        segment_footer_index_bytes_read_from_peer +=
144
0
                other.segment_footer_index_bytes_read_from_peer;
145
0
        segment_footer_index_local_io_timer += other.segment_footer_index_local_io_timer;
146
0
        segment_footer_index_remote_io_timer += other.segment_footer_index_remote_io_timer;
147
0
        segment_footer_index_peer_io_timer += other.segment_footer_index_peer_io_timer;
148
149
0
        num_cross_cg_peer_io_total += other.num_cross_cg_peer_io_total;
150
0
        bytes_read_from_cross_cg_peer += other.bytes_read_from_cross_cg_peer;
151
0
        cross_cg_peer_io_timer += other.cross_cg_peer_io_timer;
152
0
        num_same_cg_peer_io_total += other.num_same_cg_peer_io_total;
153
0
        bytes_read_from_same_cg_peer += other.bytes_read_from_same_cg_peer;
154
0
        same_cg_peer_io_timer += other.same_cg_peer_io_timer;
155
0
        num_peer_race_peer_win += other.num_peer_race_peer_win;
156
0
        num_peer_race_s3_win += other.num_peer_race_s3_win;
157
0
        num_peer_lazy_fetch += other.num_peer_lazy_fetch;
158
0
        peer_lazy_fetch_timer += other.peer_lazy_fetch_timer;
159
160
0
        peer_hosts.insert(other.peer_hosts.begin(), other.peer_hosts.end());
161
0
    }
162
};
163
164
struct IOContext {
165
    ReaderType reader_type = ReaderType::UNKNOWN;
166
    // FIXME(plat1ko): Seems `is_disposable` can be inferred from the `reader_type`?
167
    bool is_disposable = false;
168
    bool is_index_data = false;
169
    bool read_file_cache = true;
170
    // TODO(lightman): use following member variables to control file cache
171
    bool is_persistent = false;
172
    // stop reader when reading, used in some interrupted operations
173
    bool should_stop = false;
174
    int64_t expiration_time = 0;
175
    const TUniqueId* query_id = nullptr;             // Ref
176
    FileCacheStatistics* file_cache_stats = nullptr; // Ref
177
    FileReaderStats* file_reader_stats = nullptr;    // Ref
178
    bool is_inverted_index = false;
179
    // if is_dryrun, read IO will download data to cache but return no data to reader
180
    // useful to skip cache data read from local disk to accelarate warm up
181
    bool is_dryrun = false;
182
    // if `is_warmup` == true, this I/O request is from a warm up task
183
    bool is_warmup {false};
184
    int64_t condition_cache_filtered_rows = 0;
185
    // Rows removed by file-local predicate conjuncts inside FileReader/TableReader. Scanner-level
186
    // output filtering already records its own unselected rows; this counter carries the rows that
187
    // were filtered before the block returned to Scanner.
188
    int64_t predicate_filtered_rows = 0;
189
    // if true, bypass peer read / peer-vs-S3 race and read directly from remote storage
190
    bool bypass_peer_read {false};
191
};
192
193
} // namespace io
194
} // namespace doris