Coverage Report

Created: 2026-10-24 03:36

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/changelog.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <limits>
21
#include <map>
22
#include <memory>
23
#include <mutex>
24
#include <string>
25
#include <string_view>
26
#include <utility>
27
#include <vector>
28
29
#include "common/logging.h" // DCHECK
30
#include "common/status.h"
31
#include "exec/sink/autoinc_buffer.h"
32
#include "storage/changelog_types.h"
33
#include "storage/olap_common.h"
34
#include "storage/olap_define.h" // DataWriteType
35
#include "storage/rowset/rowset_fwd.h"
36
#include "storage/tablet/tablet_fwd.h"    // BaseTabletSPtr
37
#include "storage/tablet/tablet_schema.h" // TabletSchemaSPtr
38
#include "storage/utils.h"                // CHANGELOG_*_COL
39
40
namespace doris {
41
42
class AutoIncIDBuffer;
43
struct PartialUpdateInfo;
44
struct MowContext;
45
46
namespace changelog {
47
48
// ATTN(changelog-rename): formerly "__BEFORE__<col>__"
49
4.32k
inline std::string build_before_column_name(std::string_view name) {
50
4.32k
    std::string before_name = "__DORIS_CHANGELOG_BEFORE_";
51
4.32k
    before_name.append(name.data(), name.size());
52
4.32k
    before_name.append("__");
53
4.32k
    return before_name;
54
4.32k
}
55
56
} // namespace changelog
57
58
constexpr int64_t kChangelogLsnAutoIncId = -1;
59
60
// Allocate per-row LSNs.
61
// The caller must provide a valid auto-inc buffer (typically from GlobalAutoIncBuffers).
62
inline Status allocate_lsn(const std::shared_ptr<AutoIncIDBuffer>& lsn_buffer, size_t num_rows,
63
776
                           std::vector<int64_t>& lsn_ids) {
64
776
    if (lsn_buffer == nullptr) {
65
0
        return Status::InternalError("try to get lsn buffer but null");
66
0
    }
67
776
    DCHECK(num_rows > 0);
68
69
776
    std::vector<std::pair<int64_t, size_t>> ranges;
70
776
    RETURN_IF_ERROR(lsn_buffer->sync_request_ids(num_rows, &ranges));
71
72
776
    lsn_ids.clear();
73
776
    lsn_ids.reserve(num_rows);
74
776
    for (const auto& [start, length] : ranges) {
75
2.04k
        for (size_t i = 0; i < length; ++i) {
76
1.26k
            DCHECK_LE(start, std::numeric_limits<int64_t>::max() - static_cast<int64_t>(i));
77
1.26k
            lsn_ids.push_back(start + static_cast<int64_t>(i));
78
1.26k
        }
79
776
    }
80
776
    DCHECK_EQ(lsn_ids.size(), num_rows);
81
776
    return Status::OK();
82
776
}
83
84
constexpr int64_t kTsoLogicalBits = 18;
85
86
355k
inline int64_t extract_tso_physical_time(int64_t tso) {
87
355k
    return tso <= 0 ? 0 : tso >> kTsoLogicalBits;
88
355k
}
89
90
355k
inline int64_t changelog_ttl_cutoff_tso(int64_t reference_tso, int64_t ttl_seconds) {
91
355k
    DCHECK_GT(reference_tso, 0);
92
355k
    DCHECK_GE(ttl_seconds, 0);
93
355k
    const int64_t reference_ms = extract_tso_physical_time(reference_tso);
94
355k
    if (ttl_seconds > reference_ms / 1000) {
95
2
        return 0;
96
2
    }
97
355k
    constexpr int64_t kMaxLogical = (1L << kTsoLogicalBits) - 1;
98
355k
    return ((reference_ms - ttl_seconds * 1000) << kTsoLogicalBits) | kMaxLogical;
99
355k
}
100
101
namespace segment_v2 {
102
103
class SegmentAllocatedLsnMap {
104
public:
105
988
    void insert_segment_allocated_lsns(int64_t seg_id, ConstAllocatedLsnVectorSharedPtr lsn_ids) {
106
988
        std::lock_guard<std::mutex> l(_mutex);
107
988
        _seg_id_to_lsn_ids.emplace(seg_id, std::move(lsn_ids));
108
988
    }
109
110
1.90k
    void remove_segment(int64_t seg_id) {
111
1.90k
        std::lock_guard<std::mutex> l(_mutex);
112
1.90k
        _seg_id_to_lsn_ids.erase(seg_id);
113
1.90k
    }
114
115
    bool contains_segment(int64_t seg_id) const {
116
        std::lock_guard<std::mutex> l(_mutex);
117
        return _seg_id_to_lsn_ids.count(seg_id) > 0;
118
    }
119
120
973
    ConstAllocatedLsnVectorSharedPtr get_segment_allocated_lsns(int64_t seg_id) const {
121
973
        std::lock_guard<std::mutex> l(_mutex);
122
973
        auto it = _seg_id_to_lsn_ids.find(seg_id);
123
18.4E
        DCHECK(it != _seg_id_to_lsn_ids.end())
124
18.4E
                << "SegmentAllocatedLsnMap::get_segment_allocated_lsns missing seg_id=" << seg_id
125
18.4E
                << ", existing_seg_ids=[" << ([&] {
126
0
                       std::string s;
127
0
                       for (const auto& [id, _] : _seg_id_to_lsn_ids) {
128
0
                           if (!s.empty()) {
129
0
                               s.push_back(',');
130
0
                           }
131
0
                           s.append(std::to_string(id));
132
0
                       }
133
0
                       return s;
134
0
                   }())
135
18.4E
                << "]";
136
973
        return it->second;
137
973
    }
138
139
private:
140
    mutable std::mutex _mutex;
141
    std::map<int64_t, ConstAllocatedLsnVectorSharedPtr> _seg_id_to_lsn_ids;
142
};
143
144
struct SegmentWriteChangelogOptions {
145
public:
146
    bool write_before = false;
147
148
    // source context, used for retrieving historical row and building changelog block
149
    struct SourceWriteDataOptions {
150
        BaseTabletSPtr base_tablet = nullptr;
151
        TabletSchemaSPtr tablet_schema = nullptr;
152
        std::shared_ptr<PartialUpdateInfo> partial_update_info;
153
        std::shared_ptr<MowContext> mow_context;
154
        bool is_transient_rowset_writer = false;
155
        DataWriteType source_write_type = DataWriteType::TYPE_DEFAULT;
156
        // input rowset's changelog rowset, read LSN from it at publish time
157
        RowsetSharedPtr changelog_rowset = nullptr;
158
    } source;
159
};
160
} // namespace segment_v2
161
162
} // namespace doris