be/src/storage/tablet/tablet_meta_manager.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "storage/tablet/tablet_meta_manager.h"

#include <fmt/format.h>
#include <gen_cpp/olap_file.pb.h>
#include <rocksdb/db.h>
#include <rocksdb/write_batch.h>

#include <boost/algorithm/string/trim.hpp>
#include <fstream>
#include <string>
#include <unordered_set>
#include <vector>

#include "common/logging.h"
#include "common/status.h"
#include "json2pb/json_to_pb.h"
#include "json2pb/pb_to_json.h"
#include "storage/data_dir.h"
#include "storage/olap_define.h"
#include "storage/olap_meta.h"
#include "storage/utils.h"

namespace rocksdb {
class Iterator;
class Slice;
class Status;
struct ColumnFamilyOptions;
struct DBOptions;
struct ReadOptions;
struct WriteOptions;
} // namespace rocksdb
namespace doris {
using namespace ErrorCode;

// should use tablet->generate_tablet_meta_copy() method to get a copy of current tablet meta
// there are some rowset meta in local meta store and in in-memory tablet meta
// but not in tablet meta in local meta store
Status TabletMetaManager::get_meta(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
                                   TabletMetaSharedPtr tablet_meta) {
    OlapMeta* meta = store->get_meta();
    std::stringstream key_stream;
    key_stream << HEADER_PREFIX << tablet_id << "_" << schema_hash;
    std::string key = key_stream.str();
    std::string value;
    Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value);
    if (s.is<META_KEY_NOT_FOUND>()) {
        return Status::Error<META_KEY_NOT_FOUND>("tablet_id:{}, schema_hash:{}, not found.",
                                                 tablet_id, schema_hash);
    } else if (!s.ok()) {
        LOG(WARNING) << "load tablet_id:" << tablet_id << ", schema_hash:" << schema_hash
                     << " failed.";
        return s;
    }
    return tablet_meta->deserialize(value);
}

Status TabletMetaManager::get_json_meta(DataDir* store, TTabletId tablet_id,
                                        TSchemaHash schema_hash, std::string* json_meta) {
    TabletMetaSharedPtr tablet_meta(new TabletMeta());
    Status s = get_meta(store, tablet_id, schema_hash, tablet_meta);
    if (!s.ok()) {
        return s;
    }
    json2pb::Pb2JsonOptions json_options;
    json_options.pretty_json = true;
    tablet_meta->to_json(json_meta, json_options);
    return Status::OK();
}

// TODO(ygl):
// 1. if term > 0 then save to remote meta store first using term
// 2. save to local meta store
Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
                               TabletMetaSharedPtr tablet_meta, std::string_view header_prefix) {
    std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
    std::string value;
    tablet_meta->serialize(&value);
    if (tablet_meta->partition_id() <= 0) {
        LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << " tablet "
                     << tablet_meta->tablet_id();
        // TODO(dx): after fix partition id eq 0 bug, fix it
        // return Status::InternalError("invaid partition id {} tablet {}",
        //  tablet_meta->partition_id(), tablet_meta->tablet_id());
    }
    OlapMeta* meta = store->get_meta();
    VLOG_NOTICE << "save tablet meta"
                << ", key:" << key << ", meta length:" << value.length();
    return meta->put(META_COLUMN_FAMILY_INDEX, key, value);
}

Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
                               const std::string& meta_binary, std::string_view header_prefix) {
    std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
    OlapMeta* meta = store->get_meta();
    VLOG_NOTICE << "save tablet meta "
                << ", key:" << key << " meta_size=" << meta_binary.length();
    return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
}

// TODO(ygl):
// 1. remove load data first
// 2. remove from load meta store using term if term > 0
Status TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
                                 std::string_view header_prefix) {
    std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
    OlapMeta* meta = store->get_meta();
    Status res = meta->remove(META_COLUMN_FAMILY_INDEX, key);
    VLOG_NOTICE << "remove tablet_meta, key:" << key << ", res:" << res;
    return res;
}

Status TabletMetaManager::traverse_headers(
        OlapMeta* meta, std::function<bool(long, long, std::string_view)> const& func,
        std::string_view header_prefix) {
    auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
        std::vector<std::string> parts;
        // old format key format: "hdr_" + tablet_id + "_" + schema_hash  0.11
        // new format key format: "tabletmeta_" + tablet_id + "_" + schema_hash  0.10
        static_cast<void>(split_string(key, '_', &parts));
        if (parts.size() != 3) {
            LOG(WARNING) << "invalid tablet_meta key:" << key << ", split size:" << parts.size();
            return true;
        }
        TTabletId tablet_id = std::stol(parts[1], nullptr, 10);
        TSchemaHash schema_hash = cast_set<int32_t>(std::stol(parts[2], nullptr, 10));
        return func(tablet_id, schema_hash, value);
    };
    Status status = meta->iterate(META_COLUMN_FAMILY_INDEX, header_prefix, traverse_header_func);
    return status;
}

Status TabletMetaManager::load_json_meta(DataDir* store, const std::string& meta_path) {
    std::ifstream infile(meta_path);
    if (!infile.is_open()) {
        return Status::InternalError("Failed to open file {}", meta_path);
    }

    std::string json_meta((std::istreambuf_iterator<char>(infile)),
                          std::istreambuf_iterator<char>());
    boost::algorithm::trim(json_meta);
    TabletMetaPB tablet_meta_pb;
    std::string error;
    bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb, &error);
    if (!ret) {
        return Status::Error<HEADER_LOAD_JSON_HEADER>("JSON to protobuf message failed: {}", error);
    }

    std::string meta_binary;
    tablet_meta_pb.SerializeToString(&meta_binary);
    TTabletId tablet_id = tablet_meta_pb.tablet_id();
    TSchemaHash schema_hash = tablet_meta_pb.schema_hash();
    return save(store, tablet_id, schema_hash, meta_binary);
}

Status TabletMetaManager::save_pending_publish_info(DataDir* store, TTabletId tablet_id,
                                                    int64_t publish_version,
                                                    const std::string& meta_binary) {
    std::string key = fmt::format("{}{}_{}", PENDING_PUBLISH_INFO, tablet_id, publish_version);
    OlapMeta* meta = store->get_meta();
    LOG(INFO) << "save pending publish rowset, key:" << key
              << " meta_size=" << meta_binary.length();
    return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
}

Status TabletMetaManager::remove_pending_publish_info(DataDir* store, TTabletId tablet_id,
                                                      int64_t publish_version) {
    std::string key = fmt::format("{}{}_{}", PENDING_PUBLISH_INFO, tablet_id, publish_version);
    OlapMeta* meta = store->get_meta();
    Status res = meta->remove(META_COLUMN_FAMILY_INDEX, key);
    LOG(INFO) << "remove pending publish_info, key:" << key << ", res:" << res;
    return res;
}

Status TabletMetaManager::traverse_pending_publish(
        OlapMeta* meta, std::function<bool(int64_t, int64_t, std::string_view)> const& func) {
    auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
        std::vector<std::string> parts;
        // key format: "ppi_" + tablet_id + "_" + publish_version
        static_cast<void>(split_string(key, '_', &parts));
        if (parts.size() != 3) {
            LOG(WARNING) << "invalid pending publish info key:" << key
                         << ", split size:" << parts.size();
            return true;
        }
        int64_t tablet_id = std::stol(parts[1], nullptr, 10);
        int64_t version = std::stol(parts[2], nullptr, 10);
        return func(tablet_id, version, value);
    };
    Status status =
            meta->iterate(META_COLUMN_FAMILY_INDEX, PENDING_PUBLISH_INFO, traverse_header_func);
    return status;
}

std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id, int64_t version) {
    std::string key;
    key.reserve(20);
    key.append(DELETE_BITMAP);
    put_fixed64_le(&key, to_endian<std::endian::big>(tablet_id));
    put_fixed64_le(&key, to_endian<std::endian::big>(version));
    return key;
}

std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id) {
    std::string key;
    key.reserve(12);
    key.append(DELETE_BITMAP);
    put_fixed64_le(&key, to_endian<std::endian::big>(tablet_id));
    return key;
}

void NO_SANITIZE_UNDEFINED TabletMetaManager::decode_delete_bitmap_key(std::string_view enc_key,
                                                                       TTabletId* tablet_id,
                                                                       int64_t* version) {
    DCHECK_EQ(enc_key.size(), 20);
    *tablet_id = to_endian<std::endian::big>(unaligned_load<uint64_t>(enc_key.data() + 4));
    *version = to_endian<std::endian::big>(unaligned_load<uint64_t>(enc_key.data() + 12));
}

Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id,
                                             DeleteBitmapPtr delete_bitmap, int64_t version) {
    return save_delete_bitmap(store, tablet_id, std::move(delete_bitmap), nullptr, version);
}

Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id,
                                             DeleteBitmapPtr delete_bitmap,
                                             DeleteBitmapPtr binlog_delvec, int64_t version) {
    VLOG_NOTICE << "save delete bitmap, tablet_id:" << tablet_id << ", version: " << version;
    if ((delete_bitmap == nullptr || delete_bitmap->delete_bitmap.empty()) &&
        (binlog_delvec == nullptr || binlog_delvec->delete_bitmap.empty())) {
        return Status::OK();
    }
    OlapMeta* meta = store->get_meta();
    DeleteBitmapPB delete_bitmap_pb;

    // Normal delete bitmap.
    if (delete_bitmap != nullptr) {
        for (auto& [id, bitmap] : delete_bitmap->delete_bitmap) {
            auto& rowset_id = std::get<0>(id);
            auto segment_id = std::get<1>(id);
            delete_bitmap_pb.add_rowset_ids(rowset_id.to_string());
            delete_bitmap_pb.add_segment_ids(segment_id);
            std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
            bitmap.write(bitmap_data.data());
            *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data);
            delete_bitmap_pb.add_is_binlog_delvec(false);
        }
    }

    // Binlog delvec.
    if (binlog_delvec != nullptr) {
        for (auto& [id, bitmap] : binlog_delvec->delete_bitmap) {
            auto& rowset_id = std::get<0>(id);
            auto segment_id = std::get<1>(id);
            delete_bitmap_pb.add_rowset_ids(rowset_id.to_string());
            delete_bitmap_pb.add_segment_ids(segment_id);
            std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
            bitmap.write(bitmap_data.data());
            *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data);
            delete_bitmap_pb.add_is_binlog_delvec(true);
        }
    }

    std::string key = encode_delete_bitmap_key(tablet_id, version);
    std::string val;
    bool ok = delete_bitmap_pb.SerializeToString(&val);
    if (!ok) {
        auto msg = fmt::format("failed to serialize delete bitmap, tablet_id: {}, version: {}",
                               tablet_id, version);
        LOG(WARNING) << msg;
        return Status::InternalError(msg);
    }
    return meta->put(META_COLUMN_FAMILY_INDEX, key, val);
}

Status TabletMetaManager::traverse_delete_bitmap(
        OlapMeta* meta, std::function<bool(int64_t, int64_t, std::string_view)> const& func) {
    auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
        TTabletId tablet_id;
        int64_t version;
        decode_delete_bitmap_key(key, &tablet_id, &version);
        VLOG_NOTICE << "traverse delete bitmap, tablet_id: " << tablet_id
                    << ", version: " << version;
        return func(tablet_id, version, value);
    };
    return meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);
}

Status TabletMetaManager::remove_old_version_delete_bitmap(DataDir* store, TTabletId tablet_id,
                                                           int64_t version) {
    OlapMeta* meta = store->get_meta();
    std::string begin_key = encode_delete_bitmap_key(tablet_id);
    std::string end_key = encode_delete_bitmap_key(tablet_id, version);

    std::vector<std::string> remove_keys;
    auto get_remove_keys_func = [&](std::string_view key, std::string_view val) -> bool {
        // include end_key
        if (key > end_key) {
            return false;
        }
        remove_keys.emplace_back(key);
        return true;
    };
    LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
              << " version: " << version << ", removed keys size: " << remove_keys.size();
    RETURN_IF_ERROR(meta->iterate(META_COLUMN_FAMILY_INDEX, begin_key, get_remove_keys_func));
    return meta->remove(META_COLUMN_FAMILY_INDEX, remove_keys);
}

} // namespace doris

Coverage Report

Created: 2026-07-07 22:17

Line	Count	Source
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17
18		#include "storage/tablet/tablet_meta_manager.h"
19
20		#include <fmt/format.h>
21		#include <gen_cpp/olap_file.pb.h>
22		#include <rocksdb/db.h>
23		#include <rocksdb/write_batch.h>
24
25		#include <boost/algorithm/string/trim.hpp>
26		#include <fstream>
27		#include <string>
28		#include <unordered_set>
29		#include <vector>
30
31		#include "common/logging.h"
32		#include "common/status.h"
33		#include "json2pb/json_to_pb.h"
34		#include "json2pb/pb_to_json.h"
35		#include "storage/data_dir.h"
36		#include "storage/olap_define.h"
37		#include "storage/olap_meta.h"
38		#include "storage/utils.h"
39
40		namespace rocksdb {
41		class Iterator;
42		class Slice;
43		class Status;
44		struct ColumnFamilyOptions;
45		struct DBOptions;
46		struct ReadOptions;
47		struct WriteOptions;
48		} // namespace rocksdb
49		namespace doris {
50		using namespace ErrorCode;
51
52		// should use tablet->generate_tablet_meta_copy() method to get a copy of current tablet meta
53		// there are some rowset meta in local meta store and in in-memory tablet meta
54		// but not in tablet meta in local meta store
55		Status TabletMetaManager::get_meta(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
56	486	TabletMetaSharedPtr tablet_meta) {
57	486	OlapMeta* meta = store->get_meta();
58	486	std::stringstream key_stream;
59	486	key_stream << HEADER_PREFIX << tablet_id << "_" << schema_hash;
60	486	std::string key = key_stream.str();
61	486	std::string value;
62	486	Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value);
63	486	if (s.is<META_KEY_NOT_FOUND>()) {
64	22	return Status::Error<META_KEY_NOT_FOUND>("tablet_id:{}, schema_hash:{}, not found.",
65	22	tablet_id, schema_hash);
66	464	} else if (!s.ok()) {
67	0	LOG(WARNING) << "load tablet_id:" << tablet_id << ", schema_hash:" << schema_hash
68	0	<< " failed.";
69	0	return s;
70	0	}
71	464	return tablet_meta->deserialize(value);
72	486	}
73
74		Status TabletMetaManager::get_json_meta(DataDir* store, TTabletId tablet_id,
75	4	TSchemaHash schema_hash, std::string* json_meta) {
76	4	TabletMetaSharedPtr tablet_meta(new TabletMeta());
77	4	Status s = get_meta(store, tablet_id, schema_hash, tablet_meta);
78	4	if (!s.ok()) {
79	0	return s;
80	0	}
81	4	json2pb::Pb2JsonOptions json_options;
82	4	json_options.pretty_json = true;
83	4	tablet_meta->to_json(json_meta, json_options);
84	4	return Status::OK();
85	4	}
86
87		// TODO(ygl):
88		// 1. if term > 0 then save to remote meta store first using term
89		// 2. save to local meta store
90		Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
91	2	TabletMetaSharedPtr tablet_meta, std::string_view header_prefix) {
92	2	std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
93	2	std::string value;
94	2	tablet_meta->serialize(&value);
95	2	if (tablet_meta->partition_id() <= 0) {
96	0	LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << " tablet "
97	0	<< tablet_meta->tablet_id();
98		// TODO(dx): after fix partition id eq 0 bug, fix it
99		// return Status::InternalError("invaid partition id {} tablet {}",
100		// tablet_meta->partition_id(), tablet_meta->tablet_id());
101	0	}
102	2	OlapMeta* meta = store->get_meta();
103	2	VLOG_NOTICE << "save tablet meta"
104	2	<< ", key:" << key << ", meta length:" << value.length();
105	2	return meta->put(META_COLUMN_FAMILY_INDEX, key, value);
106	2	}
107
108		Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
109	780	const std::string& meta_binary, std::string_view header_prefix) {
110	780	std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
111	780	OlapMeta* meta = store->get_meta();
112	780	VLOG_NOTICE << "save tablet meta "
113	580	<< ", key:" << key << " meta_size=" << meta_binary.length();
114	780	return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
115	780	}
116
117		// TODO(ygl):
118		// 1. remove load data first
119		// 2. remove from load meta store using term if term > 0
120		Status TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
121	228	std::string_view header_prefix) {
122	228	std::string key = fmt::format("{}{}_{}", header_prefix, tablet_id, schema_hash);
123	228	OlapMeta* meta = store->get_meta();
124	228	Status res = meta->remove(META_COLUMN_FAMILY_INDEX, key);
125	228	VLOG_NOTICE << "remove tablet_meta, key:" << key << ", res:" << res;
126	228	return res;
127	228	}
128
129		Status TabletMetaManager::traverse_headers(
130		OlapMeta* meta, std::function<bool(long, long, std::string_view)> const& func,
131	204	std::string_view header_prefix) {
132	204	auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
133	0	std::vector<std::string> parts;
134		// old format key format: "hdr_" + tablet_id + "_" + schema_hash 0.11
135		// new format key format: "tabletmeta_" + tablet_id + "_" + schema_hash 0.10
136	0	static_cast<void>(split_string(key, '_', &parts));
137	0	if (parts.size() != 3) {
138	0	LOG(WARNING) << "invalid tablet_meta key:" << key << ", split size:" << parts.size();
139	0	return true;
140	0	}
141	0	TTabletId tablet_id = std::stol(parts[1], nullptr, 10);
142	0	TSchemaHash schema_hash = cast_set<int32_t>(std::stol(parts[2], nullptr, 10));
143	0	return func(tablet_id, schema_hash, value);
144	0	};
145	204	Status status = meta->iterate(META_COLUMN_FAMILY_INDEX, header_prefix, traverse_header_func);
146	204	return status;
147	204	}
148
149	4	Status TabletMetaManager::load_json_meta(DataDir* store, const std::string& meta_path) {
150	4	std::ifstream infile(meta_path);
151	4	if (!infile.is_open()) {
152	2	return Status::InternalError("Failed to open file {}", meta_path);
153	2	}
154
155	2	std::string json_meta((std::istreambuf_iterator<char>(infile)),
156	2	std::istreambuf_iterator<char>());
157	2	boost::algorithm::trim(json_meta);
158	2	TabletMetaPB tablet_meta_pb;
159	2	std::string error;
160	2	bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb, &error);
161	2	if (!ret) {
162	0	return Status::Error<HEADER_LOAD_JSON_HEADER>("JSON to protobuf message failed: {}", error);
163	0	}
164
165	2	std::string meta_binary;
166	2	tablet_meta_pb.SerializeToString(&meta_binary);
167	2	TTabletId tablet_id = tablet_meta_pb.tablet_id();
168	2	TSchemaHash schema_hash = tablet_meta_pb.schema_hash();
169	2	return save(store, tablet_id, schema_hash, meta_binary);
170	2	}
171
172		Status TabletMetaManager::save_pending_publish_info(DataDir* store, TTabletId tablet_id,
173		int64_t publish_version,
174	4.07k	const std::string& meta_binary) {
175	4.07k	std::string key = fmt::format("{}{}_{}", PENDING_PUBLISH_INFO, tablet_id, publish_version);
176	4.07k	OlapMeta* meta = store->get_meta();
177	4.07k	LOG(INFO) << "save pending publish rowset, key:" << key
178	4.07k	<< " meta_size=" << meta_binary.length();
179	4.07k	return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
180	4.07k	}
181
182		Status TabletMetaManager::remove_pending_publish_info(DataDir* store, TTabletId tablet_id,
183	74	int64_t publish_version) {
184	74	std::string key = fmt::format("{}{}_{}", PENDING_PUBLISH_INFO, tablet_id, publish_version);
185	74	OlapMeta* meta = store->get_meta();
186	74	Status res = meta->remove(META_COLUMN_FAMILY_INDEX, key);
187	74	LOG(INFO) << "remove pending publish_info, key:" << key << ", res:" << res;
188	74	return res;
189	74	}
190
191		Status TabletMetaManager::traverse_pending_publish(
192	104	OlapMeta* meta, std::function<bool(int64_t, int64_t, std::string_view)> const& func) {
193	104	auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
194	14	std::vector<std::string> parts;
195		// key format: "ppi_" + tablet_id + "_" + publish_version
196	14	static_cast<void>(split_string(key, '_', &parts));
197	14	if (parts.size() != 3) {
198	0	LOG(WARNING) << "invalid pending publish info key:" << key
199	0	<< ", split size:" << parts.size();
200	0	return true;
201	0	}
202	14	int64_t tablet_id = std::stol(parts[1], nullptr, 10);
203	14	int64_t version = std::stol(parts[2], nullptr, 10);
204	14	return func(tablet_id, version, value);
205	14	};
206	104	Status status =
207	104	meta->iterate(META_COLUMN_FAMILY_INDEX, PENDING_PUBLISH_INFO, traverse_header_func);
208	104	return status;
209	104	}
210
211	610	std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id, int64_t version) {
212	610	std::string key;
213	610	key.reserve(20);
214	610	key.append(DELETE_BITMAP);
215	610	put_fixed64_le(&key, to_endian<std::endian::big>(tablet_id));
216	610	put_fixed64_le(&key, to_endian<std::endian::big>(version));
217	610	return key;
218	610	}
219
220	6	std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id) {
221	6	std::string key;
222	6	key.reserve(12);
223	6	key.append(DELETE_BITMAP);
224	6	put_fixed64_le(&key, to_endian<std::endian::big>(tablet_id));
225	6	return key;
226	6	}
227
228		void NO_SANITIZE_UNDEFINED TabletMetaManager::decode_delete_bitmap_key(std::string_view enc_key,
229		TTabletId* tablet_id,
230	1.19k	int64_t* version) {
231	1.19k	DCHECK_EQ(enc_key.size(), 20);
232	1.19k	*tablet_id = to_endian<std::endian::big>(unaligned_load<uint64_t>(enc_key.data() + 4));
233	1.19k	*version = to_endian<std::endian::big>(unaligned_load<uint64_t>(enc_key.data() + 12));
234	1.19k	}
235
236		Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id,
237	600	DeleteBitmapPtr delete_bitmap, int64_t version) {
238	600	return save_delete_bitmap(store, tablet_id, std::move(delete_bitmap), nullptr, version);
239	600	}
240
241		Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id,
242		DeleteBitmapPtr delete_bitmap,
243	606	DeleteBitmapPtr binlog_delvec, int64_t version) {
244	606	VLOG_NOTICE << "save delete bitmap, tablet_id:" << tablet_id << ", version: " << version;
245	606	if ((delete_bitmap == nullptr \|\| delete_bitmap->delete_bitmap.empty()) &&
246	606	(binlog_delvec == nullptr \|\| binlog_delvec->delete_bitmap.empty())) {
247	4	return Status::OK();
248	4	}
249	602	OlapMeta* meta = store->get_meta();
250	602	DeleteBitmapPB delete_bitmap_pb;
251
252		// Normal delete bitmap.
253	602	if (delete_bitmap != nullptr) {
254	15.0k	for (auto& [id, bitmap] : delete_bitmap->delete_bitmap) {
255	15.0k	auto& rowset_id = std::get<0>(id);
256	15.0k	auto segment_id = std::get<1>(id);
257	15.0k	delete_bitmap_pb.add_rowset_ids(rowset_id.to_string());
258	15.0k	delete_bitmap_pb.add_segment_ids(segment_id);
259	15.0k	std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
260	15.0k	bitmap.write(bitmap_data.data());
261	15.0k	*(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data);
262	15.0k	delete_bitmap_pb.add_is_binlog_delvec(false);
263	15.0k	}
264	602	}
265
266		// Binlog delvec.
267	602	if (binlog_delvec != nullptr) {
268	0	for (auto& [id, bitmap] : binlog_delvec->delete_bitmap) {
269	0	auto& rowset_id = std::get<0>(id);
270	0	auto segment_id = std::get<1>(id);
271	0	delete_bitmap_pb.add_rowset_ids(rowset_id.to_string());
272	0	delete_bitmap_pb.add_segment_ids(segment_id);
273	0	std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
274	0	bitmap.write(bitmap_data.data());
275	0	*(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data);
276	0	delete_bitmap_pb.add_is_binlog_delvec(true);
277	0	}
278	0	}
279
280	602	std::string key = encode_delete_bitmap_key(tablet_id, version);
281	602	std::string val;
282	602	bool ok = delete_bitmap_pb.SerializeToString(&val);
283	602	if (!ok) {
284	0	auto msg = fmt::format("failed to serialize delete bitmap, tablet_id: {}, version: {}",
285	0	tablet_id, version);
286	0	LOG(WARNING) << msg;
287	0	return Status::InternalError(msg);
288	0	}
289	602	return meta->put(META_COLUMN_FAMILY_INDEX, key, val);
290	602	}
291
292		Status TabletMetaManager::traverse_delete_bitmap(
293	110	OlapMeta* meta, std::function<bool(int64_t, int64_t, std::string_view)> const& func) {
294	1.19k	auto traverse_header_func = [&func](std::string_view key, std::string_view value) -> bool {
295	1.19k	TTabletId tablet_id;
296	1.19k	int64_t version;
297	1.19k	decode_delete_bitmap_key(key, &tablet_id, &version);
298	1.19k	VLOG_NOTICE << "traverse delete bitmap, tablet_id: " << tablet_id
299	1.19k	<< ", version: " << version;
300	1.19k	return func(tablet_id, version, value);
301	1.19k	};
302	110	return meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);
303	110	}
304
305		Status TabletMetaManager::remove_old_version_delete_bitmap(DataDir* store, TTabletId tablet_id,
306	6	int64_t version) {
307	6	OlapMeta* meta = store->get_meta();
308	6	std::string begin_key = encode_delete_bitmap_key(tablet_id);
309	6	std::string end_key = encode_delete_bitmap_key(tablet_id, version);
310
311	6	std::vector<std::string> remove_keys;
312	604	auto get_remove_keys_func = [&](std::string_view key, std::string_view val) -> bool {
313		// include end_key
314	604	if (key > end_key) {
315	4	return false;
316	4	}
317	600	remove_keys.emplace_back(key);
318	600	return true;
319	604	};
320	6	LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
321	6	<< " version: " << version << ", removed keys size: " << remove_keys.size();
322	6	RETURN_IF_ERROR(meta->iterate(META_COLUMN_FAMILY_INDEX, begin_key, get_remove_keys_func));
323	6	return meta->remove(META_COLUMN_FAMILY_INDEX, remove_keys);
324	6	}
325
326		} // namespace doris