Coverage Report

Created: 2024-11-20 16:51

/root/doris/be/src/olap/rowid_conversion.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <map>
21
#include <vector>
22
23
#include "olap/olap_common.h"
24
#include "olap/utils.h"
25
#include "runtime/thread_context.h"
26
27
namespace doris {
28
29
// For unique key merge on write table, we should update delete bitmap
30
// of destination rowset when compaction finished.
31
// Through the row id correspondence between the source rowset and the
32
// destination rowset, we can quickly update the delete bitmap of the
33
// destination rowset.
34
class RowIdConversion {
35
public:
36
113
    RowIdConversion() = default;
37
113
    ~RowIdConversion() { RELEASE_THREAD_MEM_TRACKER(_seg_rowid_map_mem_used); }
38
39
    // resize segment rowid map to its rows num
40
292
    void init_segment_map(const RowsetId& src_rowset_id, const std::vector<uint32_t>& num_rows) {
41
292
        size_t delta_std_pair_cap = 0;
42
734
        for (size_t i = 0; i < num_rows.size(); i++) {
43
442
            uint32_t id = _segments_rowid_map.size();
44
442
            _segment_to_id_map.emplace(std::pair<RowsetId, uint32_t> {src_rowset_id, i}, id);
45
442
            _id_to_segment_map.emplace_back(src_rowset_id, i);
46
442
            std::vector<std::pair<uint32_t, uint32_t>> vec(
47
442
                    num_rows[i], std::pair<uint32_t, uint32_t>(UINT32_MAX, UINT32_MAX));
48
442
            delta_std_pair_cap += vec.capacity();
49
442
            _segments_rowid_map.emplace_back(std::move(vec));
50
442
        }
51
        //NOTE: manually count _segments_rowid_map's memory here, because _segments_rowid_map could be used by indexCompaction.
52
        // indexCompaction is a thridparty code, it's too complex to modify it.
53
        // refer compact_column.
54
292
        track_mem_usage(delta_std_pair_cap);
55
292
    }
56
57
    // set dst rowset id
58
98
    void set_dst_rowset_id(const RowsetId& dst_rowset_id) { _dst_rowst_id = dst_rowset_id; }
59
2
    const RowsetId get_dst_rowset_id() { return _dst_rowst_id; }
60
61
    // add row id to the map
62
    void add(const std::vector<RowLocation>& rss_row_ids,
63
5.21k
             const std::vector<uint32_t>& dst_segments_num_row) {
64
4.68M
        for (auto& item : rss_row_ids) {
65
4.68M
            if (item.row_id == -1) {
66
0
                continue;
67
0
            }
68
4.68M
            uint32_t id = _segment_to_id_map.at(
69
4.68M
                    std::pair<RowsetId, uint32_t> {item.rowset_id, item.segment_id});
70
4.68M
            if (_cur_dst_segment_id < dst_segments_num_row.size() &&
71
4.68M
                _cur_dst_segment_rowid >= dst_segments_num_row[_cur_dst_segment_id]) {
72
3.63k
                _cur_dst_segment_id++;
73
3.63k
                _cur_dst_segment_rowid = 0;
74
3.63k
            }
75
4.68M
            _segments_rowid_map[id][item.row_id] =
76
4.68M
                    std::pair<uint32_t, uint32_t> {_cur_dst_segment_id, _cur_dst_segment_rowid++};
77
4.68M
        }
78
5.21k
    }
79
80
    // get destination RowLocation
81
    // return non-zero if the src RowLocation does not exist
82
1.75M
    int get(const RowLocation& src, RowLocation* dst) const {
83
1.75M
        auto iter = _segment_to_id_map.find({src.rowset_id, src.segment_id});
84
1.75M
        if (iter == _segment_to_id_map.end()) {
85
1
            return -1;
86
1
        }
87
1.75M
        const auto& rowid_map = _segments_rowid_map[iter->second];
88
1.75M
        if (src.row_id >= rowid_map.size()) {
89
1
            return -1;
90
1
        }
91
1.75M
        auto& [dst_segment_id, dst_rowid] = rowid_map[src.row_id];
92
1.75M
        if (dst_segment_id == UINT32_MAX && dst_rowid == UINT32_MAX) {
93
764k
            return -1;
94
764k
        }
95
96
993k
        dst->rowset_id = _dst_rowst_id;
97
993k
        dst->segment_id = dst_segment_id;
98
993k
        dst->row_id = dst_rowid;
99
993k
        return 0;
100
1.75M
    }
101
102
    const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& get_rowid_conversion_map()
103
2
            const {
104
2
        return _segments_rowid_map;
105
2
    }
106
107
2
    const std::map<std::pair<RowsetId, uint32_t>, uint32_t>& get_src_segment_to_id_map() {
108
2
        return _segment_to_id_map;
109
2
    }
110
111
0
    std::pair<RowsetId, uint32_t> get_segment_by_id(uint32_t id) const {
112
0
        DCHECK_GT(_id_to_segment_map.size(), id);
113
0
        return _id_to_segment_map.at(id);
114
0
    }
115
116
0
    uint32_t get_id_by_segment(const std::pair<RowsetId, uint32_t>& segment) const {
117
0
        return _segment_to_id_map.at(segment);
118
0
    }
119
120
private:
121
292
    void track_mem_usage(size_t delta_std_pair_cap) {
122
292
        _std_pair_cap += delta_std_pair_cap;
123
124
292
        size_t new_size =
125
292
                _std_pair_cap * sizeof(std::pair<uint32_t, uint32_t>) +
126
292
                _segments_rowid_map.capacity() * sizeof(std::vector<std::pair<uint32_t, uint32_t>>);
127
128
292
        RELEASE_THREAD_MEM_TRACKER(_seg_rowid_map_mem_used);
129
292
        CONSUME_THREAD_MEM_TRACKER(new_size);
130
292
        _seg_rowid_map_mem_used = new_size;
131
292
    }
132
133
private:
134
    // the first level vector: index indicates src segment.
135
    // the second level vector: index indicates row id of source segment,
136
    // value indicates row id of destination segment.
137
    // <UINT32_MAX, UINT32_MAX> indicates current row not exist.
138
    std::vector<std::vector<std::pair<uint32_t, uint32_t>>> _segments_rowid_map;
139
    size_t _seg_rowid_map_mem_used {0};
140
    size_t _std_pair_cap {0};
141
142
    // Map source segment to 0 to n
143
    std::map<std::pair<RowsetId, uint32_t>, uint32_t> _segment_to_id_map;
144
145
    // Map 0 to n to source segment
146
    std::vector<std::pair<RowsetId, uint32_t>> _id_to_segment_map;
147
148
    // dst rowset id
149
    RowsetId _dst_rowst_id;
150
151
    // current dst segment id
152
    std::uint32_t _cur_dst_segment_id = 0;
153
154
    // current rowid of dst segment
155
    std::uint32_t _cur_dst_segment_rowid = 0;
156
};
157
158
} // namespace doris