be/src/storage/snapshot/snapshot_manager.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "storage/snapshot/snapshot_manager.h"

#include <fmt/format.h>
#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/Types_constants.h>
#include <gen_cpp/olap_file.pb.h>
#include <thrift/protocol/TDebugProtocol.h>

#include <algorithm>
#include <atomic>
#include <ctime>
#include <filesystem>
#include <list>
#include <map>
#include <new>
#include <ostream>
#include <set>
#include <shared_mutex>
#include <unordered_map>
#include <utility>

#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
#include "io/fs/local_file_system.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/thread_context.h"
#include "storage/data_dir.h"
#include "storage/olap_common.h"
#include "storage/olap_define.h"
#include "storage/pb_helper.h"
#include "storage/rowset/rowset.h"
#include "storage/rowset/rowset_factory.h"
#include "storage/rowset/rowset_meta.h"
#include "storage/rowset/rowset_writer.h"
#include "storage/rowset/rowset_writer_context.h"
#include "storage/storage_engine.h"
#include "storage/tablet/tablet_manager.h"
#include "storage/tablet/tablet_meta.h"
#include "storage/tablet/tablet_schema.h"
#include "storage/tablet/tablet_schema_cache.h"
#include "storage/utils.h"
#include "util/uid_util.h"

using std::nothrow;
using std::string;
using std::stringstream;
using std::vector;

namespace doris {
using namespace ErrorCode;

LocalSnapshotLockGuard LocalSnapshotLock::acquire(const std::string& path) {
    std::unique_lock<std::mutex> l(_lock);
    auto& ctx = _local_snapshot_contexts[path];
    while (ctx._is_locked) {
        ctx._waiting_count++;
        ctx._cv.wait(l);
        ctx._waiting_count--;
    }

    ctx._is_locked = true;
    return {path};
}

void LocalSnapshotLock::release(const std::string& path) {
    std::lock_guard<std::mutex> l(_lock);
    auto iter = _local_snapshot_contexts.find(path);
    if (iter == _local_snapshot_contexts.end()) {
        return;
    }

    auto& ctx = iter->second;
    ctx._is_locked = false;
    if (ctx._waiting_count > 0) {
        ctx._cv.notify_one();
    } else {
        _local_snapshot_contexts.erase(iter);
    }
}

SnapshotManager::SnapshotManager(StorageEngine& engine) : _engine(engine) {
    _mem_tracker =
            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "SnapshotManager");
}

SnapshotManager::~SnapshotManager() = default;

Status SnapshotManager::make_snapshot(const TSnapshotRequest& request, string* snapshot_path,
                                      bool* allow_incremental_clone) {
    SCOPED_ATTACH_TASK(_mem_tracker);
    Status res = Status::OK();
    if (snapshot_path == nullptr) {
        return Status::Error<INVALID_ARGUMENT>("output parameter cannot be null");
    }

    TabletSharedPtr target_tablet = _engine.tablet_manager()->get_tablet(request.tablet_id);

    DBUG_EXECUTE_IF("SnapshotManager::make_snapshot.inject_failure", { target_tablet = nullptr; })

    if (target_tablet == nullptr) {
        return Status::Error<TABLE_NOT_FOUND>("failed to get tablet. tablet={}", request.tablet_id);
    }

    TabletSharedPtr ref_tablet = target_tablet;
    if (request.__isset.ref_tablet_id) {
        int64_t ref_tablet_id = request.ref_tablet_id;
        TabletSharedPtr base_tablet = _engine.tablet_manager()->get_tablet(ref_tablet_id);

        // Some tasks, like medium migration, cause the target tablet and base tablet to stay on
        // different disks. In this case, we fall through to the normal restore path.
        //
        // Otherwise, we can directly link the rowset files from the base tablet to the target tablet.
        if (base_tablet != nullptr &&
            base_tablet->data_dir()->path() == target_tablet->data_dir()->path()) {
            ref_tablet = std::move(base_tablet);
        }
    }

    res = _create_snapshot_files(ref_tablet, target_tablet, request, snapshot_path,
                                 allow_incremental_clone);

    if (!res.ok()) {
        LOG(WARNING) << "failed to make snapshot. res=" << res << " tablet=" << request.tablet_id;
        return res;
    }

    LOG(INFO) << "success to make snapshot. path=['" << *snapshot_path << "']";
    return res;
}

Status SnapshotManager::release_snapshot(const string& snapshot_path) {
    auto local_snapshot_guard = LocalSnapshotLock::instance().acquire(snapshot_path);

    // If the requested snapshot_path is located in the root/snapshot folder, it is considered legal and can be deleted.
    // Otherwise, it is considered an illegal request and returns an error result.
    SCOPED_ATTACH_TASK(_mem_tracker);
    auto stores = _engine.get_stores();
    for (auto* store : stores) {
        std::string abs_path;
        RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize(store->path(), &abs_path));
        if (snapshot_path.compare(0, abs_path.size(), abs_path) == 0 &&
            snapshot_path.compare(abs_path.size() + 1, SNAPSHOT_PREFIX.size(), SNAPSHOT_PREFIX) ==
                    0) {
            RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(snapshot_path));
            LOG(INFO) << "success to release snapshot path. [path='" << snapshot_path << "']";
            return Status::OK();
        }
    }

    return Status::Error<CE_CMD_PARAMS_ERROR>("released snapshot path illegal. [path='{}']",
                                              snapshot_path);
}

Result<std::vector<PendingRowsetGuard>> SnapshotManager::convert_rowset_ids(
        const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id,
        int64_t partition_id, int32_t schema_hash) {
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker);
    std::vector<PendingRowsetGuard> guards;
    // check clone dir existed
    bool exists = true;
    RETURN_IF_ERROR_RESULT(io::global_local_filesystem()->exists(clone_dir, &exists));
    if (!exists) {
        return unexpected(Status::Error<DIR_NOT_EXIST>(
                "clone dir not existed when convert rowsetids. clone_dir={}", clone_dir));
    }

    TabletSharedPtr target_tablet = _engine.tablet_manager()->get_tablet(tablet_id);
    TabletSchemaSPtr target_tablet_schema = nullptr;
    if (target_tablet != nullptr) {
        target_tablet_schema = std::make_shared<TabletSchema>();
        target_tablet_schema->copy_from(*target_tablet->tablet_schema());
    }

    // load original tablet meta
    auto cloned_meta_file = fmt::format("{}/{}.hdr", clone_dir, tablet_id);
    TabletMetaPB cloned_tablet_meta_pb;
    RETURN_IF_ERROR_RESULT(TabletMeta::load_from_file(cloned_meta_file, &cloned_tablet_meta_pb));

    TabletMetaPB new_tablet_meta_pb;
    new_tablet_meta_pb = cloned_tablet_meta_pb;
    new_tablet_meta_pb.clear_rs_metas();
    // inc_rs_meta is deprecated since 0.13.
    // keep this just for safety
    new_tablet_meta_pb.clear_inc_rs_metas();
    new_tablet_meta_pb.clear_stale_rs_metas();
    // should modify tablet id and schema hash because in restore process the tablet id is not
    // equal to tablet id in meta
    new_tablet_meta_pb.set_tablet_id(tablet_id);
    *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto();
    new_tablet_meta_pb.set_replica_id(replica_id);
    if (table_id > 0) {
        new_tablet_meta_pb.set_table_id(table_id);
    }
    if (partition_id != -1) {
        new_tablet_meta_pb.set_partition_id(partition_id);
    }
    new_tablet_meta_pb.set_schema_hash(schema_hash);
    TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
    tablet_schema->init_from_pb(new_tablet_meta_pb.schema());

    std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> rs_version_map;
    std::unordered_map<RowsetId, RowsetId> rowset_id_mapping;
    guards.reserve(cloned_tablet_meta_pb.rs_metas_size() +
                   cloned_tablet_meta_pb.stale_rs_metas_size());
    for (auto&& visible_rowset : cloned_tablet_meta_pb.rs_metas()) {
        RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_rs_metas();
        if (!visible_rowset.has_resource_id()) {
            // src be local rowset
            RowsetId rowset_id = _engine.next_rowset_id();
            guards.push_back(_engine.pending_local_rowsets().add(rowset_id));
            RETURN_IF_ERROR_RESULT(_rename_rowset_id(
                    visible_rowset, clone_dir, tablet_schema, rowset_id, rowset_meta,
                    new_tablet_meta_pb.enable_unique_key_merge_on_write()));
            RowsetId src_rs_id;
            if (visible_rowset.rowset_id() > 0) {
                src_rs_id.init(visible_rowset.rowset_id());
            } else {
                src_rs_id.init(visible_rowset.rowset_id_v2());
            }
            rowset_id_mapping[src_rs_id] = rowset_id;
            rowset_meta->set_source_rowset_id(visible_rowset.rowset_id_v2());
            rowset_meta->set_source_tablet_id(cloned_tablet_meta_pb.tablet_id());
        } else {
            // remote rowset
            *rowset_meta = visible_rowset;
        }

        rowset_meta->set_tablet_id(tablet_id);
        if (partition_id != -1) {
            rowset_meta->set_partition_id(partition_id);
        }

        if (rowset_meta->has_tablet_schema() && rowset_meta->tablet_schema().index_size() > 0) {
            RETURN_IF_ERROR_RESULT(
                    _rename_index_ids(*rowset_meta->mutable_tablet_schema(), target_tablet_schema));
        }

        Version rowset_version = {visible_rowset.start_version(), visible_rowset.end_version()};
        rs_version_map[rowset_version] = rowset_meta;
    }

    for (auto&& stale_rowset : cloned_tablet_meta_pb.stale_rs_metas()) {
        Version rowset_version = {stale_rowset.start_version(), stale_rowset.end_version()};
        auto exist_rs = rs_version_map.find(rowset_version);
        if (exist_rs != rs_version_map.end()) {
            continue;
        }
        RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_stale_rs_metas();

        if (!stale_rowset.has_resource_id()) {
            // src be local rowset
            RowsetId rowset_id = _engine.next_rowset_id();
            guards.push_back(_engine.pending_local_rowsets().add(rowset_id));
            RETURN_IF_ERROR_RESULT(_rename_rowset_id(
                    stale_rowset, clone_dir, tablet_schema, rowset_id, rowset_meta,
                    new_tablet_meta_pb.enable_unique_key_merge_on_write()));
            RowsetId src_rs_id;
            if (stale_rowset.rowset_id() > 0) {
                src_rs_id.init(stale_rowset.rowset_id());
            } else {
                src_rs_id.init(stale_rowset.rowset_id_v2());
            }
            rowset_id_mapping[src_rs_id] = rowset_id;
            rowset_meta->set_source_rowset_id(stale_rowset.rowset_id_v2());
            rowset_meta->set_source_tablet_id(cloned_tablet_meta_pb.tablet_id());
        } else {
            // remote rowset
            *rowset_meta = stale_rowset;
        }

        rowset_meta->set_tablet_id(tablet_id);
        if (partition_id != -1) {
            rowset_meta->set_partition_id(partition_id);
        }

        if (rowset_meta->has_tablet_schema() && rowset_meta->tablet_schema().index_size() > 0) {
            RETURN_IF_ERROR_RESULT(
                    _rename_index_ids(*rowset_meta->mutable_tablet_schema(), target_tablet_schema));
        }
    }

    if (new_tablet_meta_pb.schema().index_size() > 0) {
        RETURN_IF_ERROR_RESULT(
                _rename_index_ids(*new_tablet_meta_pb.mutable_schema(), target_tablet_schema));
    }

    if (!rowset_id_mapping.empty() && cloned_tablet_meta_pb.has_delete_bitmap()) {
        const auto& cloned_del_bitmap_pb = cloned_tablet_meta_pb.delete_bitmap();
        DeleteBitmapPB* new_del_bitmap_pb = new_tablet_meta_pb.mutable_delete_bitmap();
        int rst_ids_size = cloned_del_bitmap_pb.rowset_ids_size();
        for (int i = 0; i < rst_ids_size; ++i) {
            RowsetId rst_id;
            rst_id.init(cloned_del_bitmap_pb.rowset_ids(i));
            // It should not happen, if we can't convert some rowid in delete bitmap, the
            // data might be inconsist.
            CHECK(rowset_id_mapping.find(rst_id) != rowset_id_mapping.end())
                    << "can't find rowset_id " << rst_id.to_string() << " in convert_rowset_ids";
            new_del_bitmap_pb->set_rowset_ids(i, rowset_id_mapping[rst_id].to_string());
        }
    }

    RETURN_IF_ERROR_RESULT(TabletMeta::save(cloned_meta_file, new_tablet_meta_pb));

    return guards;
}

Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb,
                                          const std::string& new_tablet_path,
                                          TabletSchemaSPtr tablet_schema, const RowsetId& rowset_id,
                                          RowsetMetaPB* new_rs_meta_pb,
                                          bool enable_unique_key_merge_on_write) {
    Status st = Status::OK();
    RowsetMetaSharedPtr rowset_meta(new RowsetMeta());
    rowset_meta->init_from_pb(rs_meta_pb);
    RowsetSharedPtr org_rowset;
    RETURN_IF_ERROR(
            RowsetFactory::create_rowset(tablet_schema, new_tablet_path, rowset_meta, &org_rowset));
    // do not use cache to load index
    // because the index file may conflict
    // and the cached fd may be invalid
    RETURN_IF_ERROR(org_rowset->load(false));
    RowsetMetaSharedPtr org_rowset_meta = org_rowset->rowset_meta();
    RowsetWriterContext context;
    context.rowset_id = rowset_id;
    context.tablet_id = org_rowset_meta->tablet_id();
    context.partition_id = org_rowset_meta->partition_id();
    context.tablet_schema_hash = org_rowset_meta->tablet_schema_hash();
    context.rowset_type = org_rowset_meta->rowset_type();
    context.tablet_path = new_tablet_path;
    context.tablet_schema =
            org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : tablet_schema;
    context.rowset_state = org_rowset_meta->rowset_state();
    context.version = org_rowset_meta->version();
    context.newest_write_timestamp = org_rowset_meta->newest_write_timestamp();
    // keep segments_overlap same as origin rowset
    context.segments_overlap = rowset_meta->segments_overlap();
    // propagate MOW flag so that non-MOW key-bounds aggregation is not applied
    // when restoring a MOW tablet's rowset
    context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write;

    auto rs_writer = DORIS_TRY(RowsetFactory::create_rowset_writer(_engine, context, false));

    st = rs_writer->add_rowset(org_rowset);
    if (!st.ok()) {
        LOG(WARNING) << "failed to add rowset "
                     << " id = " << org_rowset->rowset_id() << " to rowset " << rowset_id;
        return st;
    }
    RowsetSharedPtr new_rowset;
    RETURN_NOT_OK_STATUS_WITH_WARN(rs_writer->build(new_rowset),
                                   "failed to build rowset when rename rowset id");
    RETURN_IF_ERROR(new_rowset->load(false));
    new_rowset->rowset_meta()->to_rowset_pb(new_rs_meta_pb);
    RETURN_IF_ERROR(org_rowset->remove());
    return Status::OK();
}

Status SnapshotManager::_rename_index_ids(TabletSchemaPB& schema_pb,
                                          const TabletSchemaSPtr& tablet_schema) const {
    if (tablet_schema == nullptr) {
        return Status::OK();
    }

    for (int i = 0; i < schema_pb.index_size(); ++i) {
        TabletIndexPB* index_pb = schema_pb.mutable_index(i);
        for (int32_t col_unique_id : index_pb->col_unique_id()) {
            auto local_index = tablet_schema->get_index(col_unique_id, index_pb->index_type(),
                                                        index_pb->index_suffix_name());
            if (local_index) {
                if (index_pb->index_id() != local_index->index_id()) {
                    index_pb->set_index_id(local_index->index_id());
                }
                break;
            }
        }
    }
    return Status::OK();
}

// get snapshot path: curtime.seq.timeout
// eg: 20190819221234.3.86400
Status SnapshotManager::_calc_snapshot_id_path(const TabletSharedPtr& tablet, int64_t timeout_s,
                                               std::string* out_path) {
    Status res = Status::OK();
    if (out_path == nullptr) {
        return Status::Error<INVALID_ARGUMENT>("output parameter cannot be null");
    }

    // get current timestamp string
    string time_str;
    if ((res = gen_timestamp_string(&time_str)) != Status::OK()) {
        LOG(WARNING) << "failed to generate time_string when move file to trash."
                     << "err code=" << res;
        return res;
    }

    uint64_t sid = _snapshot_base_id.fetch_add(1, std::memory_order_relaxed) - 1;
    *out_path = fmt::format("{}/{}/{}.{}.{}", tablet->data_dir()->path(), SNAPSHOT_PREFIX, time_str,
                            sid, timeout_s);
    return res;
}

// prefix: /path/to/data/DATA_PREFIX/shard_id
// return: /path/to/data/DATA_PREFIX/shard_id/tablet_id/schema_hash
std::string SnapshotManager::get_schema_hash_full_path(const TabletSharedPtr& ref_tablet,
                                                       const std::string& prefix) {
    return fmt::format("{}/{}/{}", prefix, ref_tablet->tablet_id(), ref_tablet->schema_hash());
}

std::string SnapshotManager::_get_header_full_path(const TabletSharedPtr& ref_tablet,
                                                   const std::string& schema_hash_path) const {
    return fmt::format("{}/{}.hdr", schema_hash_path, ref_tablet->tablet_id());
}

std::string SnapshotManager::_get_json_header_full_path(const TabletSharedPtr& ref_tablet,
                                                        const std::string& schema_hash_path) const {
    return fmt::format("{}/{}.hdr.json", schema_hash_path, ref_tablet->tablet_id());
}

Status SnapshotManager::_link_index_and_data_files(
        const std::string& schema_hash_path, const TabletSharedPtr& ref_tablet,
        const std::vector<RowsetSharedPtr>& consistent_rowsets) {
    Status res = Status::OK();
    for (auto& rs : consistent_rowsets) {
        RETURN_IF_ERROR(rs->link_files_to(schema_hash_path, rs->rowset_id()));
    }
    return res;
}

Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet,
                                               const TabletSharedPtr& target_tablet,
                                               const TSnapshotRequest& request,
                                               string* snapshot_path,
                                               bool* allow_incremental_clone) {
    int32_t snapshot_version = request.preferred_snapshot_version;
    LOG(INFO) << "receive a make snapshot request"
              << ", request detail is " << apache::thrift::ThriftDebugString(request)
              << ", snapshot_version is " << snapshot_version;
    Status res = Status::OK();
    if (snapshot_path == nullptr) {
        return Status::Error<INVALID_ARGUMENT>("output parameter cannot be null");
    }

    // snapshot_id_path:
    //      /data/shard_id/tablet_id/snapshot/time_str/id.timeout/
    int64_t timeout_s = config::snapshot_expire_time_sec;
    if (request.__isset.timeout) {
        timeout_s = request.timeout;
    }
    std::string snapshot_id_path;
    res = _calc_snapshot_id_path(target_tablet, timeout_s, &snapshot_id_path);
    if (!res.ok()) {
        LOG(WARNING) << "failed to calc snapshot_id_path, tablet="
                     << target_tablet->data_dir()->path();
        return res;
    }

    bool is_copy_binlog = request.__isset.is_copy_binlog ? request.is_copy_binlog : false;

    // schema_full_path_desc.filepath:
    //      /snapshot_id_path/tablet_id/schema_hash/
    auto schema_full_path = get_schema_hash_full_path(target_tablet, snapshot_id_path);
    // header_path:
    //      /schema_full_path/tablet_id.hdr
    auto header_path = _get_header_full_path(target_tablet, schema_full_path);
    //      /schema_full_path/tablet_id.hdr.json
    auto json_header_path = _get_json_header_full_path(target_tablet, schema_full_path);
    bool exists = true;
    RETURN_IF_ERROR(io::global_local_filesystem()->exists(schema_full_path, &exists));
    if (exists) {
        VLOG_TRACE << "remove the old schema_full_path." << schema_full_path;
        RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(schema_full_path));
    }

    RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(schema_full_path));
    string snapshot_id;
    RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize(snapshot_id_path, &snapshot_id));

    std::vector<RowsetSharedPtr> consistent_rowsets;
    do {
        TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
        if (new_tablet_meta == nullptr) {
            res = Status::Error<MEM_ALLOC_FAILED>("fail to malloc TabletMeta.");
            break;
        }
        DeleteBitmap delete_bitmap_snapshot(new_tablet_meta->tablet_id());

        /// If set missing_version, try to get all missing version.
        /// If some of them not exist in tablet, we will fall back to
        /// make the full snapshot of the tablet.
        {
            std::shared_lock rdlock(ref_tablet->get_header_lock());
            if (ref_tablet->tablet_state() == TABLET_SHUTDOWN) {
                return Status::Aborted("tablet has shutdown");
            }
            bool is_single_rowset_clone =
                    (request.__isset.start_version && request.__isset.end_version);
            if (is_single_rowset_clone) {
                LOG(INFO) << "handle compaction clone make snapshot, tablet_id: "
                          << ref_tablet->tablet_id();
                Version version(request.start_version, request.end_version);
                const RowsetSharedPtr rowset = ref_tablet->get_rowset_by_version(version, false);
                if (rowset && rowset->is_local()) {
                    consistent_rowsets.push_back(rowset);
                } else {
                    LOG(WARNING) << "failed to find local version when do compaction snapshot. "
                                 << " tablet=" << request.tablet_id
                                 << " schema_hash=" << request.schema_hash
                                 << " version=" << version;
                    res = Status::InternalError(
                            "failed to find version when do compaction snapshot");
                    break;
                }
            }
            // be would definitely set it as true no matter has missed version or not
            // but it would take no effects on the following range loop
            if (!is_single_rowset_clone && request.__isset.missing_version) {
                for (int64_t missed_version : request.missing_version) {
                    Version version = {missed_version, missed_version};
                    // find rowset in both rs_meta and stale_rs_meta
                    const RowsetSharedPtr rowset = ref_tablet->get_rowset_by_version(version, true);
                    if (rowset != nullptr) {
                        if (!rowset->is_local()) {
                            // MUST make full snapshot to ensure `cooldown_meta_id` is consistent with the cooldowned rowsets after clone.
                            res = Status::Error<ErrorCode::INTERNAL_ERROR>(
                                    "missed version is a cooldowned rowset, must make full "
                                    "snapshot. missed_version={}, tablet_id={}",
                                    missed_version, ref_tablet->tablet_id());
                            break;
                        }
                        consistent_rowsets.push_back(rowset);
                    } else {
                        res = Status::InternalError(
                                "failed to find missed version when snapshot. tablet={}, "
                                "schema_hash={}, version={}",
                                request.tablet_id, request.schema_hash, version.to_string());
                        break;
                    }
                }
            }

            DBUG_EXECUTE_IF("SnapshotManager.create_snapshot_files.allow_inc_clone", {
                auto tablet_id = dp->param("tablet_id", 0);
                auto is_full_clone = dp->param("is_full_clone", false);
                if (ref_tablet->tablet_id() == tablet_id && is_full_clone) {
                    LOG(INFO) << "injected full clone for tabelt: " << tablet_id;
                    res = Status::InternalError("fault injection error");
                }
            });

            // be would definitely set it as true no matter has missed version or not, we could
            // just check whether the missed version is empty or not
            int64_t version = -1;
            if (!is_single_rowset_clone && (!res.ok() || request.missing_version.empty())) {
                if (!request.__isset.missing_version &&
                    ref_tablet->tablet_meta()->cooldown_meta_id().initialized()) {
                    LOG(WARNING) << "currently not support backup tablet with cooldowned remote "
                                    "data. tablet="
                                 << request.tablet_id;
                    return Status::NotSupported(
                            "currently not support backup tablet with cooldowned remote data");
                }
                /// not all missing versions are found, fall back to full snapshot.
                res = Status::OK();         // reset res
                consistent_rowsets.clear(); // reset vector

                // get latest version
                const RowsetSharedPtr last_version = ref_tablet->get_rowset_with_max_version();
                if (last_version == nullptr) {
                    res = Status::InternalError("tablet has not any version. path={}",
                                                ref_tablet->tablet_id());
                    break;
                }
                // get snapshot version, use request.version if specified
                version = last_version->end_version();
                if (request.__isset.version) {
                    if (last_version->end_version() < request.version) {
                        res = Status::Error<INVALID_ARGUMENT>(
                                "invalid make snapshot request. version={}, req_version={}",
                                last_version->version().to_string(), request.version);
                        break;
                    }
                    version = request.version;
                }
                if (ref_tablet->tablet_meta()->cooldown_meta_id().initialized()) {
                    // Tablet has cooldowned data, MUST pick consistent rowsets with continuous cooldowned version
                    // Get max cooldowned version
                    int64_t max_cooldowned_version = -1;
                    for (auto& [v, rs] : ref_tablet->rowset_map()) {
                        if (rs->is_local()) {
                            continue;
                        }
                        consistent_rowsets.push_back(rs);
                        max_cooldowned_version = std::max(max_cooldowned_version, v.second);
                    }
                    DCHECK_GE(max_cooldowned_version, 1) << "tablet_id=" << ref_tablet->tablet_id();
                    std::sort(consistent_rowsets.begin(), consistent_rowsets.end(),
                              Rowset::comparator);
                    res = check_version_continuity(consistent_rowsets);
                    if (res.ok() && max_cooldowned_version < version) {
                        // Pick consistent rowsets of remaining required version
                        auto ret = ref_tablet->capture_consistent_rowsets_unlocked(
                                {max_cooldowned_version + 1, version}, CaptureRowsetOps {});
                        if (ret) {
                            consistent_rowsets = std::move(ret->rowsets);
                        } else {
                            res = std::move(ret.error());
                        }
                    }
                } else {
                    auto ret = ref_tablet->capture_consistent_rowsets_unlocked(Version(0, version),
                                                                               CaptureRowsetOps {});
                    if (ret) {
                        consistent_rowsets = std::move(ret->rowsets);
                    } else {
                        res = std::move(ret.error());
                    }
                }
                if (!res.ok()) {
                    LOG(WARNING) << "fail to select versions to span. res=" << res;
                    break;
                }
                *allow_incremental_clone = false;
            } else {
                version = ref_tablet->max_version_unlocked();
                *allow_incremental_clone = true;
            }

            // copy the tablet meta to new_tablet_meta inside header lock
            CHECK(res.ok()) << res;
            ref_tablet->generate_tablet_meta_copy_unlocked(*new_tablet_meta, false);
            // The delete bitmap update operation and the add_inc_rowset operation is not atomic,
            // so delete bitmap may contains some data generated by invisible rowset, we should
            // get rid of these useless bitmaps when doing snapshot.
            if (ref_tablet->keys_type() == UNIQUE_KEYS &&
                ref_tablet->enable_unique_key_merge_on_write()) {
                delete_bitmap_snapshot =
                        ref_tablet->tablet_meta()->delete_bitmap().snapshot(version);
            }
        }

        std::vector<RowsetMetaSharedPtr> rs_metas;
        for (auto& rs : consistent_rowsets) {
            if (rs->is_local()) {
                // local rowset
                res = rs->link_files_to(schema_full_path, rs->rowset_id());
                if (!res.ok()) {
                    break;
                }
            }
            rs_metas.push_back(rs->rowset_meta());
            VLOG_NOTICE << "add rowset meta to clone list. "
                        << " start version " << rs->rowset_meta()->start_version()
                        << " end version " << rs->rowset_meta()->end_version() << " empty "
                        << rs->rowset_meta()->empty();
        }
        if (!res.ok()) {
            LOG(WARNING) << "fail to create hard link. path=" << snapshot_id_path
                         << " tablet=" << target_tablet->tablet_id()
                         << " ref tablet=" << ref_tablet->tablet_id();
            break;
        }

        // The inc_rs_metas is deprecated since Doris version 0.13.
        // Clear it for safety reason.
        // Whether it is incremental or full snapshot, rowset information is stored in rs_meta.
        new_tablet_meta->revise_rs_metas(std::move(rs_metas));
        if (ref_tablet->keys_type() == UNIQUE_KEYS &&
            ref_tablet->enable_unique_key_merge_on_write()) {
            new_tablet_meta->revise_delete_bitmap_unlocked(delete_bitmap_snapshot);
        }

        if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION2) {
            res = new_tablet_meta->save(header_path);
            if (res.ok() && request.__isset.is_copy_tablet_task && request.is_copy_tablet_task) {
                res = new_tablet_meta->save_as_json(json_header_path);
            }
        } else {
            res = Status::Error<INVALID_SNAPSHOT_VERSION>(
                    "snapshot_version not equal to g_Types_constants.TSNAPSHOT_REQ_VERSION2");
        }

        if (!res.ok()) {
            LOG(WARNING) << "convert rowset failed, res:" << res
                         << ", tablet:" << new_tablet_meta->tablet_id()
                         << ", schema hash:" << new_tablet_meta->schema_hash()
                         << ", snapshot_version:" << snapshot_version
                         << ", is incremental:" << request.__isset.missing_version;
            break;
        }

    } while (false);

    // link all binlog files to snapshot path
    do {
        if (!res.ok()) {
            break;
        }

        if (!is_copy_binlog || !target_tablet->is_enable_binlog()) {
            break;
        }

        RowsetBinlogMetasPB rowset_binlog_metas_pb;
        for (auto& rs : consistent_rowsets) {
            if (!rs->is_local()) {
                continue;
            }
            res = ref_tablet->get_rowset_binlog_metas(rs->version(), &rowset_binlog_metas_pb);
            if (!res.ok()) {
                break;
            }
        }
        if (!res.ok() || rowset_binlog_metas_pb.rowset_binlog_metas_size() == 0) {
            break;
        }

        // write to pb file
        auto rowset_binlog_metas_pb_filename =
                fmt::format("{}/rowset_binlog_metas.pb", schema_full_path);
        res = write_pb(rowset_binlog_metas_pb_filename, rowset_binlog_metas_pb);
        if (!res.ok()) {
            break;
        }

        for (const auto& rowset_binlog_meta : rowset_binlog_metas_pb.rowset_binlog_metas()) {
            std::string segment_file_path;
            auto num_segments = rowset_binlog_meta.num_segments();
            std::string_view rowset_id = rowset_binlog_meta.rowset_id();

            RowsetMetaPB rowset_meta_pb;
            if (!rowset_meta_pb.ParseFromString(rowset_binlog_meta.data())) {
                auto err_msg = fmt::format("fail to parse binlog meta data value:{}",
                                           rowset_binlog_meta.data());
                res = Status::InternalError(err_msg);
                LOG(WARNING) << err_msg;
                return res;
            }
            const auto& tablet_schema_pb = rowset_meta_pb.tablet_schema();
            TabletSchema tablet_schema;
            tablet_schema.init_from_pb(tablet_schema_pb);

            std::vector<string> linked_success_files;
            Defer remove_linked_files {[&]() { // clear linked files if errors happen
                if (!res.ok()) {
                    LOG(WARNING) << "will delete linked success files due to error " << res;
                    std::vector<io::Path> paths;
                    for (auto& file : linked_success_files) {
                        paths.emplace_back(file);
                        LOG(WARNING)
                                << "will delete linked success file " << file << " due to error";
                    }
                    static_cast<void>(io::global_local_filesystem()->batch_delete(paths));
                    LOG(WARNING) << "done delete linked success files due to error " << res;
                }
            }};

            // link segment files and index files
            for (int64_t segment_index = 0; segment_index < num_segments; ++segment_index) {
                segment_file_path = ref_tablet->get_segment_filepath(rowset_id, segment_index);
                auto snapshot_segment_file_path =
                        fmt::format("{}/{}_{}.binlog", schema_full_path, rowset_id, segment_index);

                res = io::global_local_filesystem()->link_file(segment_file_path,
                                                               snapshot_segment_file_path);
                if (!res.ok()) {
                    LOG(WARNING) << "fail to link binlog file. [src=" << segment_file_path
                                 << ", dest=" << snapshot_segment_file_path << "]";
                    break;
                }
                linked_success_files.push_back(snapshot_segment_file_path);

                if (tablet_schema.get_inverted_index_storage_format() ==
                    InvertedIndexStorageFormatPB::V1) {
                    for (const auto& index : tablet_schema.inverted_indexes()) {
                        auto index_id = index->index_id();
                        auto index_file = InvertedIndexDescriptor::get_index_file_path_v1(
                                InvertedIndexDescriptor::get_index_file_path_prefix(
                                        segment_file_path),
                                index_id, index->get_index_suffix());
                        auto snapshot_segment_index_file_path =
                                fmt::format("{}/{}_{}_{}.binlog-index", schema_full_path, rowset_id,
                                            segment_index, index_id);
                        VLOG_DEBUG << "link " << index_file << " to "
                                   << snapshot_segment_index_file_path;
                        res = io::global_local_filesystem()->link_file(
                                index_file, snapshot_segment_index_file_path);
                        if (!res.ok()) {
                            LOG(WARNING) << "fail to link binlog index file. [src=" << index_file
                                         << ", dest=" << snapshot_segment_index_file_path << "]";
                            break;
                        }
                        linked_success_files.push_back(snapshot_segment_index_file_path);
                    }
                } else {
                    if (tablet_schema.has_inverted_index() || tablet_schema.has_ann_index()) {
                        auto index_file = InvertedIndexDescriptor::get_index_file_path_v2(
                                InvertedIndexDescriptor::get_index_file_path_prefix(
                                        segment_file_path));
                        auto snapshot_segment_index_file_path =
                                fmt::format("{}/{}_{}.binlog-index", schema_full_path, rowset_id,
                                            segment_index);
                        VLOG_DEBUG << "link " << index_file << " to "
                                   << snapshot_segment_index_file_path;
                        res = io::global_local_filesystem()->link_file(
                                index_file, snapshot_segment_index_file_path);
                        if (!res.ok()) {
                            LOG(WARNING) << "fail to link binlog index file. [src=" << index_file
                                         << ", dest=" << snapshot_segment_index_file_path << "]";
                            break;
                        }
                        linked_success_files.push_back(snapshot_segment_index_file_path);
                    }
                }
            }

            if (!res.ok()) {
                break;
            }
        }
    } while (false);

    if (!res.ok()) {
        LOG(WARNING) << "fail to make snapshot, try to delete the snapshot path. path="
                     << snapshot_id_path.c_str();

        bool exist = true;
        RETURN_IF_ERROR(io::global_local_filesystem()->exists(snapshot_id_path, &exist));
        if (exist) {
            VLOG_NOTICE << "remove snapshot path. [path=" << snapshot_id_path << "]";
            RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(snapshot_id_path));
        }
    } else {
        *snapshot_path = snapshot_id;
    }

    return res;
}
} // namespace doris

Coverage Report

Created: 2026-04-20 20:34