Coverage Report

Created: 2026-05-25 14:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exec/rowid_fetcher.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <brpc/controller.h>
21
#include <bthread/countdown_event.h>
22
#include <gen_cpp/DataSinks_types.h>
23
#include <gen_cpp/internal_service.pb.h>
24
25
#include <memory>
26
#include <semaphore>
27
#include <utility>
28
#include <vector>
29
30
#include "common/status.h"
31
#include "core/block/block.h"
32
#include "core/data_type/data_type.h"
33
#include "storage/id_manager.h"
34
#include "storage/tablet_info.h" // DorisNodesInfo
35
36
namespace doris {
37
38
class DorisNodesInfo;
39
class RuntimeState;
40
class TupleDescriptor;
41
42
struct FileMapping;
43
struct SegKey;
44
struct SegItem;
45
struct HashOfSegKey;
46
struct IteratorKey;
47
struct IteratorItem;
48
struct HashOfIteratorKey;
49
50
2.13k
inline void fetch_callback(bthread::CountdownEvent* counter) {
51
2.13k
    Defer __defer([&] { counter->signal(); });
52
2.13k
}
53
54
template <typename T>
55
class ColumnStr;
56
using ColumnString = ColumnStr<UInt32>;
57
class MutableBlock;
58
59
// fetch rows by global rowid
60
// tablet_id/rowset_name/segment_id/ordinal_id
61
62
struct FetchOption {
63
    TupleDescriptor* desc = nullptr;
64
    RuntimeState* runtime_state = nullptr;
65
    TFetchOption t_fetch_opt;
66
};
67
68
class RowIDFetcher {
69
public:
70
0
    RowIDFetcher(FetchOption fetch_opt) : _fetch_option(std::move(fetch_opt)) {}
71
    Status init();
72
    Status fetch(const ColumnPtr& row_ids, Block* block);
73
74
private:
75
    PMultiGetRequest _init_fetch_request(const ColumnString& row_ids) const;
76
    Status _merge_rpc_results(const PMultiGetRequest& request,
77
                              const std::vector<PMultiGetResponse>& rsps,
78
                              const std::vector<brpc::Controller>& cntls, Block* output_block,
79
                              std::vector<PRowLocation>* rows_id) const;
80
81
    std::vector<std::shared_ptr<PBackendService_Stub>> _stubs;
82
    FetchOption _fetch_option;
83
};
84
85
struct RowStoreReadStruct {
86
    DataTypeSerDeSPtrs serdes;
87
    std::unordered_map<uint32_t, uint32_t> col_uid_to_idx;
88
    std::vector<std::string> default_values;
89
};
90
91
class RowIdStorageReader {
92
public:
93
    //external profile info key.
94
    static const std::string ScannersRunningTimeProfile;
95
    static const std::string InitReaderAvgTimeProfile;
96
    static const std::string GetBlockAvgTimeProfile;
97
    static const std::string FileReadLinesProfile;
98
99
    static Status read_by_rowids(const PMultiGetRequest& request, PMultiGetResponse* response);
100
    static Status read_by_rowids(const PMultiGetRequestV2& request, PMultiGetResponseV2* response);
101
102
private:
103
    struct ExternalFetchStatistics;
104
105
    static Status read_doris_format_row(
106
            const std::shared_ptr<IdFileMap>& id_file_map,
107
            const std::shared_ptr<FileMapping>& file_mapping, const std::vector<uint32_t>& row_id,
108
            std::vector<SlotDescriptor>& slots, const TabletSchema& full_read_schema,
109
            RowStoreReadStruct& row_store_read_struct, OlapReaderStatistics& stats,
110
            int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms, int64_t* acquire_segments_ms,
111
            int64_t* lookup_row_data_ms, std::unordered_map<SegKey, SegItem, HashOfSegKey>& seg_map,
112
            std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey>& iterator_map,
113
            Block& result_block);
114
115
    static Status read_batch_doris_format_row(
116
            const PRequestBlockDesc& request_block_desc, std::shared_ptr<IdFileMap> id_file_map,
117
            std::vector<SlotDescriptor>& slots, const TUniqueId& query_id, Block& result_block,
118
            OlapReaderStatistics& stats, int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms,
119
            int64_t* acquire_segments_ms, int64_t* lookup_row_data_ms);
120
121
    static Status read_batch_external_row(
122
            const uint64_t workload_group_id, const PRequestBlockDesc& request_block_desc,
123
            std::shared_ptr<IdFileMap> id_file_map, std::vector<SlotDescriptor>& slots,
124
            std::shared_ptr<FileMapping> first_file_mapping, const TUniqueId& query_id,
125
            Block& result_block, PRuntimeProfileTree* pprofile, int64_t* init_reader_avg_ms,
126
            int64_t* get_block_avg_ms, size_t* scan_range_cnt);
127
128
    static Status read_external_row_from_file_mapping(
129
            size_t idx, const std::multimap<segment_v2::rowid_t, size_t>& row_ids,
130
            const std::shared_ptr<FileMapping>& file_mapping,
131
            const std::vector<SlotDescriptor>& slots, const TUniqueId& query_id,
132
            const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks,
133
            std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
134
            std::vector<ExternalFetchStatistics>& fetch_statistics,
135
            const TFileScanRangeParams& rpc_scan_params,
136
            const std::unordered_map<std::string, int>& colname_to_slot_id,
137
            std::atomic<int>& producer_count, size_t scan_rows_count,
138
            std::counting_semaphore<>& semaphore, std::condition_variable& cv, std::mutex& mtx,
139
            TupleDescriptor& tuple_desc);
140
141
    struct ExternalFetchStatistics {
142
        int64_t init_reader_ms = 0;
143
        int64_t get_block_ms = 0;
144
        std::string file_read_bytes;
145
        std::string file_read_times;
146
    };
147
};
148
149
template <typename Func>
150
8.55k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
8.55k
    MonotonicStopWatch watch;
152
8.55k
    watch.start();
153
8.55k
    auto res = fn();
154
8.55k
    *cost += watch.elapsed_time() / 1000 / 1000;
155
8.55k
    return res;
156
8.55k
}
Unexecuted instantiation: rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader14read_by_rowidsERKNS_16PMultiGetRequestEPNS_17PMultiGetResponseEE3$_1EEDTclfp_EET_Pl
Unexecuted instantiation: rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader14read_by_rowidsERKNS_16PMultiGetRequestEPNS_17PMultiGetResponseEE3$_2EEDTclfp_EET_Pl
Unexecuted instantiation: rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader14read_by_rowidsERKNS_16PMultiGetRequestEPNS_17PMultiGetResponseEE3$_4EEDTclfp_EET_Pl
Unexecuted instantiation: rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader14read_by_rowidsERKNS_16PMultiGetRequestEPNS_17PMultiGetResponseEE3$_5EEDTclfp_EET_Pl
rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader23read_batch_external_rowEmRKNS_17PRequestBlockDescESt10shared_ptrINS_9IdFileMapEERSt6vectorINS_14SlotDescriptorESaIS9_EES5_INS_11FileMappingEERKNS_9TUniqueIdERNS_5BlockEPNS_19PRuntimeProfileTreeEPlSM_PmE3$_0EEDTclfp_EET_SM_
Line
Count
Source
150
2.06k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
2.06k
    MonotonicStopWatch watch;
152
2.06k
    watch.start();
153
2.06k
    auto res = fn();
154
2.06k
    *cost += watch.elapsed_time() / 1000 / 1000;
155
2.06k
    return res;
156
2.06k
}
rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEERNS_5BlockEE3$_1EEDTclfp_EET_SR_
Line
Count
Source
150
6
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
6
    MonotonicStopWatch watch;
152
6
    watch.start();
153
6
    auto res = fn();
154
6
    *cost += watch.elapsed_time() / 1000 / 1000;
155
6
    return res;
156
6
}
rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEERNS_5BlockEE3$_2EEDTclfp_EET_SR_
Line
Count
Source
150
6
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
6
    MonotonicStopWatch watch;
152
6
    watch.start();
153
6
    auto res = fn();
154
6
    *cost += watch.elapsed_time() / 1000 / 1000;
155
6
    return res;
156
6
}
rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEERNS_5BlockEE3$_3EEDTclfp_EET_SR_
Line
Count
Source
150
6
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
6
    MonotonicStopWatch watch;
152
6
    watch.start();
153
6
    auto res = fn();
154
6
    *cost += watch.elapsed_time() / 1000 / 1000;
155
6
    return res;
156
6
}
Unexecuted instantiation: rowid_fetcher.cpp:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEERNS_5BlockEE3$_4EEDTclfp_EET_SR_
file_scanner.cpp:_ZN5doris15scope_timer_runIZNS_11FileScanner21read_lines_from_rangeERKNS_14TFileRangeDescERKNSt7__cxx114listIlSaIlEEEPNS_5BlockERKNS_23ExternalFileMappingInfoEPlSG_E3$_0EEDTclfp_EET_SG_
Line
Count
Source
150
3.22k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
3.22k
    MonotonicStopWatch watch;
152
3.22k
    watch.start();
153
3.22k
    auto res = fn();
154
3.22k
    *cost += watch.elapsed_time() / 1000 / 1000;
155
3.22k
    return res;
156
3.22k
}
file_scanner.cpp:_ZN5doris15scope_timer_runIZNS_11FileScanner21read_lines_from_rangeERKNS_14TFileRangeDescERKNSt7__cxx114listIlSaIlEEEPNS_5BlockERKNS_23ExternalFileMappingInfoEPlSG_E3$_1EEDTclfp_EET_SG_
Line
Count
Source
150
3.24k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
151
3.24k
    MonotonicStopWatch watch;
152
3.24k
    watch.start();
153
3.24k
    auto res = fn();
154
3.24k
    *cost += watch.elapsed_time() / 1000 / 1000;
155
3.24k
    return res;
156
3.24k
}
157
} // namespace doris