Coverage Report

Created: 2026-10-21 17:10

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/storage/changelog.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <limits>
21
#include <map>
22
#include <memory>
23
#include <mutex>
24
#include <string>
25
#include <string_view>
26
#include <utility>
27
#include <vector>
28
29
#include "common/logging.h" // DCHECK
30
#include "common/status.h"
31
#include "exec/sink/autoinc_buffer.h"
32
#include "storage/olap_common.h"
33
#include "storage/olap_define.h" // DataWriteType
34
#include "storage/rowset/rowset_fwd.h"
35
#include "storage/tablet/tablet_fwd.h"    // BaseTabletSPtr
36
#include "storage/tablet/tablet_schema.h" // TabletSchemaSPtr
37
#include "storage/utils.h"                // CHANGELOG_*_COL
38
39
namespace doris {
40
41
class AutoIncIDBuffer;
42
struct PartialUpdateInfo;
43
struct MowContext;
44
45
// Changelog op type.
46
// NOTE: The value is persisted into changelog data, so keep it stable.
47
static constexpr int64_t CHANGELOG_OP_APPEND = 0;
48
static constexpr int64_t CHANGELOG_OP_UPDATE = 1;
49
static constexpr int64_t CHANGELOG_OP_DELETE = 2;
50
51
namespace changelog {
52
53
// ATTN(changelog-rename): formerly "__BEFORE__<col>__"
54
366
inline std::string build_before_column_name(std::string_view name) {
55
366
    std::string before_name = "__DORIS_CHANGELOG_BEFORE_";
56
366
    before_name.append(name.data(), name.size());
57
366
    before_name.append("__");
58
366
    return before_name;
59
366
}
60
61
} // namespace changelog
62
63
constexpr int64_t kChangelogLsnAutoIncId = -1;
64
65
// Allocate per-row LSNs.
66
// The caller must provide a valid auto-inc buffer (typically from GlobalAutoIncBuffers).
67
inline Status allocate_lsn(const std::shared_ptr<AutoIncIDBuffer>& lsn_buffer, size_t num_rows,
68
7
                           std::vector<int64_t>& lsn_ids) {
69
7
    if (lsn_buffer == nullptr) {
70
0
        return Status::InternalError("try to get lsn buffer but null");
71
0
    }
72
7
    DCHECK(num_rows > 0);
73
74
7
    std::vector<std::pair<int64_t, size_t>> ranges;
75
7
    RETURN_IF_ERROR(lsn_buffer->sync_request_ids(num_rows, &ranges));
76
77
7
    lsn_ids.clear();
78
7
    lsn_ids.reserve(num_rows);
79
7
    for (const auto& [start, length] : ranges) {
80
18
        for (size_t i = 0; i < length; ++i) {
81
11
            DCHECK_LE(start, std::numeric_limits<int64_t>::max() - static_cast<int64_t>(i));
82
11
            lsn_ids.push_back(start + static_cast<int64_t>(i));
83
11
        }
84
7
    }
85
7
    DCHECK_EQ(lsn_ids.size(), num_rows);
86
7
    return Status::OK();
87
7
}
88
89
constexpr int64_t kTsoLogicalBits = 18;
90
91
55.3k
inline int64_t extract_tso_physical_time(int64_t tso) {
92
55.3k
    return tso <= 0 ? 0 : tso >> kTsoLogicalBits;
93
55.3k
}
94
95
55.3k
inline int64_t changelog_ttl_cutoff_tso(int64_t reference_tso, int64_t ttl_seconds) {
96
55.3k
    DCHECK_GT(reference_tso, 0);
97
55.3k
    DCHECK_GE(ttl_seconds, 0);
98
55.3k
    const int64_t reference_ms = extract_tso_physical_time(reference_tso);
99
55.3k
    if (ttl_seconds > reference_ms / 1000) {
100
2
        return 0;
101
2
    }
102
55.3k
    constexpr int64_t kMaxLogical = (1L << kTsoLogicalBits) - 1;
103
55.3k
    return ((reference_ms - ttl_seconds * 1000) << kTsoLogicalBits) | kMaxLogical;
104
55.3k
}
105
106
namespace segment_v2 {
107
108
class SegmentAllocatedLsnMap {
109
public:
110
67
    void insert_segment_allocated_lsns(int64_t seg_id, ConstAllocatedLsnVectorSharedPtr lsn_ids) {
111
67
        std::lock_guard<std::mutex> l(_mutex);
112
67
        _seg_id_to_lsn_ids.emplace(seg_id, std::move(lsn_ids));
113
67
    }
114
115
55
    void remove_segment(int64_t seg_id) {
116
55
        std::lock_guard<std::mutex> l(_mutex);
117
55
        _seg_id_to_lsn_ids.erase(seg_id);
118
55
    }
119
120
    bool contains_segment(int64_t seg_id) const {
121
        std::lock_guard<std::mutex> l(_mutex);
122
        return _seg_id_to_lsn_ids.count(seg_id) > 0;
123
    }
124
125
54
    ConstAllocatedLsnVectorSharedPtr get_segment_allocated_lsns(int64_t seg_id) const {
126
54
        std::lock_guard<std::mutex> l(_mutex);
127
54
        auto it = _seg_id_to_lsn_ids.find(seg_id);
128
54
        DCHECK(it != _seg_id_to_lsn_ids.end())
129
0
                << "SegmentAllocatedLsnMap::get_segment_allocated_lsns missing seg_id=" << seg_id
130
0
                << ", existing_seg_ids=[" << ([&] {
131
0
                       std::string s;
132
0
                       for (const auto& [id, _] : _seg_id_to_lsn_ids) {
133
0
                           if (!s.empty()) {
134
0
                               s.push_back(',');
135
0
                           }
136
0
                           s.append(std::to_string(id));
137
0
                       }
138
0
                       return s;
139
0
                   }())
140
0
                << "]";
141
54
        return it->second;
142
54
    }
143
144
private:
145
    mutable std::mutex _mutex;
146
    std::map<int64_t, ConstAllocatedLsnVectorSharedPtr> _seg_id_to_lsn_ids;
147
};
148
149
struct SegmentWriteChangelogOptions {
150
public:
151
    bool write_before = false;
152
153
    // source context, used for retrieving historical row and building changelog block
154
    struct SourceWriteDataOptions {
155
        BaseTabletSPtr base_tablet = nullptr;
156
        TabletSchemaSPtr tablet_schema = nullptr;
157
        std::shared_ptr<PartialUpdateInfo> partial_update_info;
158
        std::shared_ptr<MowContext> mow_context;
159
        bool is_transient_rowset_writer = false;
160
        DataWriteType source_write_type = DataWriteType::TYPE_DEFAULT;
161
        // input rowset's changelog rowset, read LSN from it at publish time
162
        RowsetSharedPtr changelog_rowset = nullptr;
163
    } source;
164
};
165
} // namespace segment_v2
166
167
} // namespace doris