Coverage Report

Created: 2026-08-24 08:14

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exec/rowid_fetcher.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/internal_service.pb.h>
21
22
#include <condition_variable>
23
#include <functional>
24
#include <memory>
25
#include <mutex>
26
#include <semaphore>
27
#include <utility>
28
#include <vector>
29
30
#include "common/status.h"
31
#include "core/block/block.h"
32
#include "core/data_type/data_type.h"
33
#include "storage/id_manager.h"
34
35
namespace doris {
36
37
class RuntimeState;
38
class TupleDescriptor;
39
class ScannerScheduler;
40
namespace io {
41
enum class FileCacheMissPolicy : uint8_t;
42
}
43
44
struct FileMapping;
45
struct SegKey;
46
struct SegItem;
47
struct HashOfSegKey;
48
struct IteratorKey;
49
struct IteratorItem;
50
struct HashOfIteratorKey;
51
52
class MutableBlock;
53
54
struct RowStoreReadStruct {
55
976
    RowStoreReadStruct(std::string& buffer) : row_store_buffer(buffer) {};
56
    std::string& row_store_buffer;
57
    DataTypeSerDeSPtrs serdes;
58
    std::unordered_map<uint32_t, uint32_t> col_uid_to_idx;
59
    std::vector<std::string> default_values;
60
};
61
62
class RowIdStorageReader {
63
public:
64
    //external profile info key.
65
    static const std::string ScannersRunningTimeProfile;
66
    static const std::string InitReaderAvgTimeProfile;
67
    static const std::string GetBlockAvgTimeProfile;
68
    static const std::string FileReadLinesProfile;
69
    static const std::string TopNLazyMaterializationSecondPhaseLocalIOCount;
70
    static const std::string TopNLazyMaterializationSecondPhaseLocalIOBytes;
71
    static const std::string TopNLazyMaterializationSecondPhaseRemoteIOCount;
72
    static const std::string TopNLazyMaterializationSecondPhaseRemoteIOBytes;
73
    static const std::string TopNLazyMaterializationSecondPhaseSkipCacheIOCount;
74
    static const std::string TopNLazyMaterializationSecondPhaseWriteCacheBytes;
75
    static const std::string TopNLazyMaterializationSecondPhaseLocalIOTime;
76
    static const std::string TopNLazyMaterializationSecondPhaseRemoteIOTime;
77
    static const std::string TopNLazyMaterializationSecondPhaseWriteCacheIOTime;
78
    static const std::string TopNLazyMaterializationSecondPhaseRowsRead;
79
    static const std::string TopNLazyMaterializationSecondPhaseSegmentsRead;
80
81
    static Status read_by_rowids(const PMultiGetRequestV2& request, PMultiGetResponseV2* response);
82
83
private:
84
    struct ExternalFetchStatistics;
85
86
    static Status read_doris_format_row(
87
            const std::shared_ptr<IdFileMap>& id_file_map,
88
            const std::shared_ptr<FileMapping>& file_mapping, const std::vector<uint32_t>& row_id,
89
            std::vector<SlotDescriptor>& slots, const TabletSchema& full_read_schema,
90
            RowStoreReadStruct& row_store_read_struct, OlapReaderStatistics& stats,
91
            int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms, int64_t* acquire_segments_ms,
92
            int64_t* lookup_row_data_ms, std::unordered_map<SegKey, SegItem, HashOfSegKey>& seg_map,
93
            std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey>& iterator_map,
94
            io::FileCacheMissPolicy file_cache_miss_policy, Block& result_block);
95
96
    static Status read_batch_doris_format_row(
97
            const PRequestBlockDesc& request_block_desc, std::shared_ptr<IdFileMap> id_file_map,
98
            std::vector<SlotDescriptor>& slots, const TUniqueId& query_id, Block& result_block,
99
            OlapReaderStatistics& stats, int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms,
100
            int64_t* acquire_segments_ms, int64_t* lookup_row_data_ms,
101
            io::FileCacheMissPolicy file_cache_miss_policy);
102
103
    static Status read_batch_external_row(
104
            const uint64_t workload_group_id, const PRequestBlockDesc& request_block_desc,
105
            std::shared_ptr<IdFileMap> id_file_map, std::vector<SlotDescriptor>& slots,
106
            std::shared_ptr<FileMapping> first_file_mapping, const TUniqueId& query_id,
107
            Block& result_block, PRuntimeProfileTree* pprofile, int64_t* init_reader_avg_ms,
108
            int64_t* get_block_avg_ms, size_t* scan_range_cnt);
109
110
    static Status read_external_row_from_file_mapping(
111
            size_t idx, const std::multimap<segment_v2::rowid_t, size_t>& row_ids,
112
            const std::shared_ptr<FileMapping>& file_mapping,
113
            const std::vector<SlotDescriptor>& scan_slots, const TUniqueId& query_id,
114
            const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks,
115
            std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
116
            std::vector<ExternalFetchStatistics>& fetch_statistics,
117
            const TFileScanRangeParams& rpc_scan_params,
118
            const std::unordered_map<std::string, int>& colname_to_slot_id,
119
            std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc);
120
121
    static std::string source_column_key(const SlotDescriptor& slot, uint32_t column_idx);
122
123
    friend class RowIdStorageReaderTest;
124
125
    static Status submit_external_scan_tasks(ScannerScheduler* scheduler,
126
                                             std::counting_semaphore<>& semaphore,
127
                                             size_t task_count,
128
                                             const std::function<std::string(size_t)>& make_task_id,
129
                                             const std::function<Status(size_t)>& run_task);
130
131
    struct ExternalFetchStatistics {
132
        int64_t init_reader_ms = 0;
133
        int64_t get_block_ms = 0;
134
        std::string file_read_bytes;
135
        std::string file_read_times;
136
    };
137
};
138
139
template <typename Func>
140
15.7k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
15.7k
    MonotonicStopWatch watch;
142
15.7k
    watch.start();
143
15.7k
    auto res = fn();
144
15.7k
    *cost += watch.elapsed_time() / 1000 / 1000;
145
15.7k
    return res;
146
15.7k
}
unity_9_cxx.cxx:_ZN5doris15scope_timer_runIZNS_11FileScanner21read_lines_from_rangeERKNS_14TFileRangeDescERKNSt7__cxx114listIlSaIlEEEPNS_5BlockERKNS_23ExternalFileMappingInfoEPlSG_E3$_0EEDTclfp_EET_SG_
Line
Count
Source
140
41
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
41
    MonotonicStopWatch watch;
142
41
    watch.start();
143
41
    auto res = fn();
144
41
    *cost += watch.elapsed_time() / 1000 / 1000;
145
41
    return res;
146
41
}
unity_9_cxx.cxx:_ZN5doris15scope_timer_runIZNS_11FileScanner21read_lines_from_rangeERKNS_14TFileRangeDescERKNSt7__cxx114listIlSaIlEEEPNS_5BlockERKNS_23ExternalFileMappingInfoEPlSG_E3$_1EEDTclfp_EET_SG_
Line
Count
Source
140
41
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
41
    MonotonicStopWatch watch;
142
41
    watch.start();
143
41
    auto res = fn();
144
41
    *cost += watch.elapsed_time() / 1000 / 1000;
145
41
    return res;
146
41
}
unity_8_cxx.cxx:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader23read_batch_external_rowEmRKNS_17PRequestBlockDescESt10shared_ptrINS_9IdFileMapEERSt6vectorINS_14SlotDescriptorESaIS9_EES5_INS_11FileMappingEERKNS_9TUniqueIdERNS_5BlockEPNS_19PRuntimeProfileTreeEPlSM_PmE3$_0EEDTclfp_EET_SM_
Line
Count
Source
140
16
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
16
    MonotonicStopWatch watch;
142
16
    watch.start();
143
16
    auto res = fn();
144
16
    *cost += watch.elapsed_time() / 1000 / 1000;
145
16
    return res;
146
16
}
unity_8_cxx.cxx:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEENS_2io19FileCacheMissPolicyERNS_5BlockEE3$_1EEDTclfp_EET_SR_
Line
Count
Source
140
3.75k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
3.75k
    MonotonicStopWatch watch;
142
3.75k
    watch.start();
143
3.75k
    auto res = fn();
144
3.75k
    *cost += watch.elapsed_time() / 1000 / 1000;
145
3.75k
    return res;
146
3.75k
}
unity_8_cxx.cxx:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEENS_2io19FileCacheMissPolicyERNS_5BlockEE3$_2EEDTclfp_EET_SR_
Line
Count
Source
140
3.75k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
3.75k
    MonotonicStopWatch watch;
142
3.75k
    watch.start();
143
3.75k
    auto res = fn();
144
3.75k
    *cost += watch.elapsed_time() / 1000 / 1000;
145
3.75k
    return res;
146
3.75k
}
unity_8_cxx.cxx:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEENS_2io19FileCacheMissPolicyERNS_5BlockEE3$_3EEDTclfp_EET_SR_
Line
Count
Source
140
3.75k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
3.75k
    MonotonicStopWatch watch;
142
3.75k
    watch.start();
143
3.75k
    auto res = fn();
144
3.75k
    *cost += watch.elapsed_time() / 1000 / 1000;
145
3.75k
    return res;
146
3.75k
}
unity_8_cxx.cxx:_ZN5doris15scope_timer_runIZNS_18RowIdStorageReader21read_doris_format_rowERKSt10shared_ptrINS_9IdFileMapEERKS2_INS_11FileMappingEERKSt6vectorIjSaIjEERSB_INS_14SlotDescriptorESaISG_EERKNS_12TabletSchemaERNS_18RowStoreReadStructERNS_20OlapReaderStatisticsEPlSR_SR_SR_RSt13unordered_mapINS_6SegKeyENS_7SegItemENS_12HashOfSegKeyESt8equal_toIST_ESaISt4pairIKST_SU_EEERSS_INS_11IteratorKeyENS_12IteratorItemENS_17HashOfIteratorKeyESW_IS14_ESaISY_IKS14_S15_EEENS_2io19FileCacheMissPolicyERNS_5BlockEE3$_4EEDTclfp_EET_SR_
Line
Count
Source
140
4.44k
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
141
4.44k
    MonotonicStopWatch watch;
142
4.44k
    watch.start();
143
4.44k
    auto res = fn();
144
4.44k
    *cost += watch.elapsed_time() / 1000 / 1000;
145
4.44k
    return res;
146
4.44k
}
147
} // namespace doris