be/src/storage/tablet/tablet_schema.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "storage/tablet/tablet_schema.h"

#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <glog/logging.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>

#include <algorithm>
#include <cctype>
// IWYU pragma: no_include <bits/std_abs.h>
#include <cmath> // IWYU pragma: keep
#include <memory>
#include <ostream>
#include <vector>

#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/consts.h"
#include "common/status.h"
#include "core/block/block.h"
#include "core/column/column_nothing.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_factory.hpp"
#include "core/string_ref.h"
#include "exec/common/hex.h"
#include "exprs/aggregate/aggregate_function_simple_factory.h"
#include "exprs/aggregate/aggregate_function_state_union.h"
#include "storage/index/index_writer.h" // IndexColumnWriter::check_support_*_index
#include "storage/index/inverted/analyzer/analyzer.h"
#include "storage/index/inverted/inverted_index_parser.h"
#include "storage/olap_common.h"
#include "storage/olap_define.h"
#include "storage/tablet/tablet_column_object_pool.h"
#include "storage/tablet/tablet_meta.h"
#include "storage/tablet_info.h"
#include "storage/types.h"
#include "storage/utils.h"
#include "util/json/path_in_data.h"

namespace doris {
FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) {
    std::string upper_type_str = type_str;
    std::transform(type_str.begin(), type_str.end(), upper_type_str.begin(),
                   [](auto c) { return std::toupper(c); });
    FieldType type;

    if (0 == upper_type_str.compare("TINYINT")) {
        type = FieldType::OLAP_FIELD_TYPE_TINYINT;
    } else if (0 == upper_type_str.compare("SMALLINT")) {
        type = FieldType::OLAP_FIELD_TYPE_SMALLINT;
    } else if (0 == upper_type_str.compare("INT")) {
        type = FieldType::OLAP_FIELD_TYPE_INT;
    } else if (0 == upper_type_str.compare("BIGINT")) {
        type = FieldType::OLAP_FIELD_TYPE_BIGINT;
    } else if (0 == upper_type_str.compare("LARGEINT")) {
        type = FieldType::OLAP_FIELD_TYPE_LARGEINT;
    } else if (0 == upper_type_str.compare("UNSIGNED_TINYINT")) {
        type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT;
    } else if (0 == upper_type_str.compare("UNSIGNED_SMALLINT")) {
        type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT;
    } else if (0 == upper_type_str.compare("UNSIGNED_INT")) {
        type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT;
    } else if (0 == upper_type_str.compare("UNSIGNED_BIGINT")) {
        type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT;
    } else if (0 == upper_type_str.compare("IPV4")) {
        type = FieldType::OLAP_FIELD_TYPE_IPV4;
    } else if (0 == upper_type_str.compare("IPV6")) {
        type = FieldType::OLAP_FIELD_TYPE_IPV6;
    } else if (0 == upper_type_str.compare("FLOAT")) {
        type = FieldType::OLAP_FIELD_TYPE_FLOAT;
    } else if (0 == upper_type_str.compare("DISCRETE_DOUBLE")) {
        type = FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE;
    } else if (0 == upper_type_str.compare("DOUBLE")) {
        type = FieldType::OLAP_FIELD_TYPE_DOUBLE;
    } else if (0 == upper_type_str.compare("CHAR")) {
        type = FieldType::OLAP_FIELD_TYPE_CHAR;
    } else if (0 == upper_type_str.compare("DATE")) {
        type = FieldType::OLAP_FIELD_TYPE_DATE;
    } else if (0 == upper_type_str.compare("DATEV2")) {
        type = FieldType::OLAP_FIELD_TYPE_DATEV2;
    } else if (0 == upper_type_str.compare("DATETIMEV2")) {
        type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
    } else if (0 == upper_type_str.compare("DATETIME")) {
        type = FieldType::OLAP_FIELD_TYPE_DATETIME;
    } else if (0 == upper_type_str.compare("TIMESTAMPTZ")) {
        type = FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ;
    } else if (0 == upper_type_str.compare("DECIMAL32")) {
        type = FieldType::OLAP_FIELD_TYPE_DECIMAL32;
    } else if (0 == upper_type_str.compare("DECIMAL64")) {
        type = FieldType::OLAP_FIELD_TYPE_DECIMAL64;
    } else if (0 == upper_type_str.compare("DECIMAL128I")) {
        type = FieldType::OLAP_FIELD_TYPE_DECIMAL128I;
    } else if (0 == upper_type_str.compare("DECIMAL256")) {
        type = FieldType::OLAP_FIELD_TYPE_DECIMAL256;
    } else if (0 == upper_type_str.compare(0, 7, "DECIMAL")) {
        // Keep this generic prefix match after all specific DECIMAL types; otherwise DECIMAL32,
        // DECIMAL64, DECIMAL128I, and DECIMAL256 would all be classified as DECIMAL.
        type = FieldType::OLAP_FIELD_TYPE_DECIMAL;
    } else if (0 == upper_type_str.compare(0, 7, "VARCHAR")) {
        type = FieldType::OLAP_FIELD_TYPE_VARCHAR;
    } else if (0 == upper_type_str.compare("STRING")) {
        type = FieldType::OLAP_FIELD_TYPE_STRING;
    } else if (0 == upper_type_str.compare("JSONB")) {
        type = FieldType::OLAP_FIELD_TYPE_JSONB;
    } else if (0 == upper_type_str.compare("VARIANT")) {
        type = FieldType::OLAP_FIELD_TYPE_VARIANT;
    } else if (0 == upper_type_str.compare("BOOLEAN")) {
        type = FieldType::OLAP_FIELD_TYPE_BOOL;
    } else if (0 == upper_type_str.compare(0, 3, "HLL")) {
        type = FieldType::OLAP_FIELD_TYPE_HLL;
    } else if (0 == upper_type_str.compare("STRUCT")) {
        type = FieldType::OLAP_FIELD_TYPE_STRUCT;
    } else if (0 == upper_type_str.compare("LIST")) {
        type = FieldType::OLAP_FIELD_TYPE_ARRAY;
    } else if (0 == upper_type_str.compare("MAP")) {
        type = FieldType::OLAP_FIELD_TYPE_MAP;
    } else if (0 == upper_type_str.compare("OBJECT")) {
        type = FieldType::OLAP_FIELD_TYPE_BITMAP;
    } else if (0 == upper_type_str.compare("BITMAP")) {
        type = FieldType::OLAP_FIELD_TYPE_BITMAP;
    } else if (0 == upper_type_str.compare("ARRAY")) {
        type = FieldType::OLAP_FIELD_TYPE_ARRAY;
    } else if (0 == upper_type_str.compare("QUANTILE_STATE")) {
        type = FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE;
    } else if (0 == upper_type_str.compare("AGG_STATE")) {
        type = FieldType::OLAP_FIELD_TYPE_AGG_STATE;
    } else {
        LOG(WARNING) << "invalid type string. [type='" << type_str << "']";
        type = FieldType::OLAP_FIELD_TYPE_UNKNOWN;
    }

    return type;
}

FieldAggregationMethod TabletColumn::get_aggregation_type_by_string(const std::string& str) {
    std::string upper_str = str;
    std::transform(str.begin(), str.end(), upper_str.begin(),
                   [](auto c) { return std::toupper(c); });
    FieldAggregationMethod aggregation_type;

    if (0 == upper_str.compare("NONE")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE;
    } else if (0 == upper_str.compare("SUM")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_SUM;
    } else if (0 == upper_str.compare("MIN")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MIN;
    } else if (0 == upper_str.compare("MAX")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MAX;
    } else if (0 == upper_str.compare("REPLACE")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE;
    } else if (0 == upper_str.compare("REPLACE_IF_NOT_NULL")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL;
    } else if (0 == upper_str.compare("HLL_UNION")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_HLL_UNION;
    } else if (0 == upper_str.compare("BITMAP_UNION")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_BITMAP_UNION;
    } else if (0 == upper_str.compare("QUANTILE_UNION")) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_QUANTILE_UNION;
    } else if (!upper_str.empty()) {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_GENERIC;
    } else {
        aggregation_type = FieldAggregationMethod::OLAP_FIELD_AGGREGATION_UNKNOWN;
    }

    return aggregation_type;
}

std::string TabletColumn::get_string_by_field_type(FieldType type) {
    switch (type) {
    case FieldType::OLAP_FIELD_TYPE_TINYINT:
        return "TINYINT";

    case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT:
        return "UNSIGNED_TINYINT";

    case FieldType::OLAP_FIELD_TYPE_SMALLINT:
        return "SMALLINT";

    case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT:
        return "UNSIGNED_SMALLINT";

    case FieldType::OLAP_FIELD_TYPE_INT:
        return "INT";

    case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT:
        return "UNSIGNED_INT";

    case FieldType::OLAP_FIELD_TYPE_BIGINT:
        return "BIGINT";

    case FieldType::OLAP_FIELD_TYPE_LARGEINT:
        return "LARGEINT";

    case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
        return "UNSIGNED_BIGINT";

    case FieldType::OLAP_FIELD_TYPE_IPV4:
        return "IPV4";

    case FieldType::OLAP_FIELD_TYPE_IPV6:
        return "IPV6";

    case FieldType::OLAP_FIELD_TYPE_FLOAT:
        return "FLOAT";

    case FieldType::OLAP_FIELD_TYPE_DOUBLE:
        return "DOUBLE";

    case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE:
        return "DISCRETE_DOUBLE";

    case FieldType::OLAP_FIELD_TYPE_CHAR:
        return "CHAR";

    case FieldType::OLAP_FIELD_TYPE_DATE:
        return "DATE";

    case FieldType::OLAP_FIELD_TYPE_DATEV2:
        return "DATEV2";

    case FieldType::OLAP_FIELD_TYPE_DATETIME:
        return "DATETIME";

    case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
        return "DATETIMEV2";

    case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
        return "TIMESTAMPTZ";

    case FieldType::OLAP_FIELD_TYPE_DECIMAL:
        return "DECIMAL";

    case FieldType::OLAP_FIELD_TYPE_DECIMAL32:
        return "DECIMAL32";

    case FieldType::OLAP_FIELD_TYPE_DECIMAL64:
        return "DECIMAL64";

    case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
        return "DECIMAL128I";

    case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
        return "DECIMAL256";

    case FieldType::OLAP_FIELD_TYPE_VARCHAR:
        return "VARCHAR";

    case FieldType::OLAP_FIELD_TYPE_JSONB:
        return "JSONB";

    case FieldType::OLAP_FIELD_TYPE_VARIANT:
        return "VARIANT";

    case FieldType::OLAP_FIELD_TYPE_STRING:
        return "STRING";

    case FieldType::OLAP_FIELD_TYPE_BOOL:
        return "BOOLEAN";

    case FieldType::OLAP_FIELD_TYPE_HLL:
        return "HLL";

    case FieldType::OLAP_FIELD_TYPE_STRUCT:
        return "STRUCT";

    case FieldType::OLAP_FIELD_TYPE_ARRAY:
        return "ARRAY";

    case FieldType::OLAP_FIELD_TYPE_MAP:
        return "MAP";

    case FieldType::OLAP_FIELD_TYPE_BITMAP:
        return "OBJECT";
    case FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE:
        return "QUANTILE_STATE";
    case FieldType::OLAP_FIELD_TYPE_AGG_STATE:
        return "AGG_STATE";
    default:
        return "UNKNOWN";
    }
}

std::string TabletColumn::get_string_by_aggregation_type(FieldAggregationMethod type) {
    switch (type) {
    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE:
        return "NONE";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_SUM:
        return "SUM";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MIN:
        return "MIN";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MAX:
        return "MAX";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE:
        return "REPLACE";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL:
        return "REPLACE_IF_NOT_NULL";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_HLL_UNION:
        return "HLL_UNION";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_BITMAP_UNION:
        return "BITMAP_UNION";

    case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_QUANTILE_UNION:
        return "QUANTILE_UNION";

    default:
        return "UNKNOWN";
    }
}

uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint32_t string_length) {
    switch (type) {
    case TPrimitiveType::TINYINT:
    case TPrimitiveType::BOOLEAN:
        return 1;
    case TPrimitiveType::SMALLINT:
        return 2;
    case TPrimitiveType::INT:
        return 4;
    case TPrimitiveType::BIGINT:
        return 8;
    case TPrimitiveType::LARGEINT:
        return 16;
    case TPrimitiveType::IPV4:
        return 4;
    case TPrimitiveType::IPV6:
        return 16;
    case TPrimitiveType::DATE:
        return 3;
    case TPrimitiveType::DATEV2:
        return 4;
    case TPrimitiveType::DATETIME:
        return 8;
    case TPrimitiveType::DATETIMEV2:
    case TPrimitiveType::TIMESTAMPTZ:
        return 8;
    case TPrimitiveType::FLOAT:
        return 4;
    case TPrimitiveType::DOUBLE:
        return 8;
    case TPrimitiveType::QUANTILE_STATE:
    case TPrimitiveType::BITMAP:
        return 16;
    case TPrimitiveType::CHAR:
        return string_length;
    case TPrimitiveType::VARCHAR:
    case TPrimitiveType::HLL:
    case TPrimitiveType::AGG_STATE:
        return string_length + sizeof(OLAP_VARCHAR_MAX_LENGTH);
    case TPrimitiveType::STRING:
    case TPrimitiveType::VARIANT:
        return string_length + sizeof(OLAP_STRING_MAX_LENGTH);
    case TPrimitiveType::JSONB:
        return string_length + sizeof(OLAP_JSONB_MAX_LENGTH);
    case TPrimitiveType::STRUCT:
        // Note that(xy): this is the length of struct type itself,
        // the length of its subtypes are not included.
        return OLAP_STRUCT_MAX_LENGTH;
    case TPrimitiveType::ARRAY:
        return OLAP_ARRAY_MAX_LENGTH;
    case TPrimitiveType::MAP:
        return OLAP_MAP_MAX_LENGTH;
    case TPrimitiveType::DECIMAL32:
        return 4;
    case TPrimitiveType::DECIMAL64:
        return 8;
    case TPrimitiveType::DECIMAL128I:
        return 16;
    case TPrimitiveType::DECIMAL256:
        return 32;
    case TPrimitiveType::DECIMALV2:
        return 12; // use 12 bytes in olap engine.
    default:
        LOG(WARNING) << "unknown field type. [type=" << type << "]";
        return 0;
    }
}

bool TabletColumn::has_char_type() const {
    switch (_type) {
    case FieldType::OLAP_FIELD_TYPE_CHAR: {
        return true;
    }
    case FieldType::OLAP_FIELD_TYPE_ARRAY:
    case FieldType::OLAP_FIELD_TYPE_MAP:
    case FieldType::OLAP_FIELD_TYPE_STRUCT: {
        return std::any_of(_sub_columns.begin(), _sub_columns.end(),
                           [&](const auto& sub) -> bool { return sub->has_char_type(); });
    }
    default:
        return false;
    }
}

TabletColumn::TabletColumn() : _aggregation(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) {}

TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType type) {
    _aggregation = agg;
    _type = type;
}

TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable) {
    _aggregation = agg;
    _type = filed_type;
    _length = cast_set<int32_t>(field_type_size(filed_type));
    _is_nullable = is_nullable;
}

TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable,
                           int32_t unique_id, size_t length) {
    _aggregation = agg;
    _type = filed_type;
    _is_nullable = is_nullable;
    _unique_id = unique_id;
    _length = cast_set<int32_t>(length);
}

TabletColumn::TabletColumn(const ColumnPB& column) {
    init_from_pb(column);
}

TabletColumn::TabletColumn(const TColumn& column) {
    init_from_thrift(column);
}

void TabletColumn::init_from_thrift(const TColumn& tcolumn) {
    ColumnPB column_pb;
    TabletMeta::init_column_from_tcolumn(tcolumn.col_unique_id, tcolumn, &column_pb);
    init_from_pb(column_pb);
}

void TabletColumn::init_from_pb(const ColumnPB& column) {
    _unique_id = column.unique_id();
    _col_name = column.name();
    _col_name_lower_case = to_lower(_col_name);
    _type = TabletColumn::get_field_type_by_string(column.type());
    _is_key = column.is_key();
    _is_nullable = column.is_nullable();
    _is_auto_increment = column.is_auto_increment();
    if (column.has_is_on_update_current_timestamp()) {
        _is_on_update_current_timestamp = column.is_on_update_current_timestamp();
    }

    _has_default_value = column.has_default_value();
    if (_has_default_value) {
        _default_value = column.default_value();
    }

    if (column.has_precision()) {
        _is_decimal = true;
        _precision = column.precision();
    } else {
        _is_decimal = false;
    }
    if (column.has_frac()) {
        _frac = column.frac();
    }
    _length = column.length();
    _index_length = column.index_length();
    if (column.has_is_bf_column()) {
        _is_bf_column = column.is_bf_column();
    } else {
        _is_bf_column = false;
    }
    if (column.has_aggregation()) {
        _aggregation = get_aggregation_type_by_string(column.aggregation());
        _aggregation_name = column.aggregation();
    }

    if (_type == FieldType::OLAP_FIELD_TYPE_AGG_STATE) {
        _result_is_nullable = column.result_is_nullable();
        _be_exec_version = column.be_exec_version();
    }

    if (column.has_visible()) {
        _visible = column.visible();
    }
    if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
        CHECK(column.children_columns_size() == 1)
                << "ARRAY type should has 1 children types, but got "
                << column.children_columns_size();
    }
    if (_type == FieldType::OLAP_FIELD_TYPE_MAP) {
        DCHECK(column.children_columns_size() == 2)
                << "MAP type should has 2 children types, but got "
                << column.children_columns_size();
        if (UNLIKELY(column.children_columns_size() != 2)) {
            LOG(WARNING) << "MAP type should has 2 children types, but got "
                         << column.children_columns_size();
        }
    }
    for (int i = 0; i < column.children_columns_size(); i++) {
        TabletColumn child_column;
        child_column.init_from_pb(column.children_columns(i));
        add_sub_column(child_column);
    }
    if (column.has_column_path_info()) {
        _column_path = std::make_shared<PathInData>();
        _column_path->from_protobuf(column.column_path_info());
        _parent_col_unique_id = column.column_path_info().parrent_column_unique_id();
    }
    if (is_variant_type() && !column.has_column_path_info()) {
        // set path info for variant root column, to prevent from missing
        _column_path = std::make_shared<PathInData>(_col_name_lower_case);
        // _parent_col_unique_id = _unique_id;
    }
    if (column.has_variant_max_subcolumns_count()) {
        _variant.max_subcolumns_count = column.variant_max_subcolumns_count();
    }
    if (column.has_variant_enable_typed_paths_to_sparse()) {
        _variant.enable_typed_paths_to_sparse = column.variant_enable_typed_paths_to_sparse();
    }
    if (column.has_variant_max_sparse_column_statistics_size()) {
        _variant.max_sparse_column_statistics_size =
                column.variant_max_sparse_column_statistics_size();
    }
    if (column.has_variant_sparse_hash_shard_count()) {
        _variant.sparse_hash_shard_count = column.variant_sparse_hash_shard_count();
    }
    if (column.has_variant_enable_doc_mode()) {
        _variant.enable_doc_mode = column.variant_enable_doc_mode();
    }
    if (column.has_variant_doc_materialization_min_rows()) {
        _variant.doc_materialization_min_rows = column.variant_doc_materialization_min_rows();
    }
    if (column.has_variant_doc_hash_shard_count()) {
        _variant.doc_hash_shard_count = column.variant_doc_hash_shard_count();
    }
    if (column.has_variant_enable_nested_group()) {
        _variant.enable_nested_group = column.variant_enable_nested_group();
    }
    if (column.has_pattern_type()) {
        _pattern_type = column.pattern_type();
    }
}

void TabletColumn::to_schema_pb(ColumnPB* column) const {
    column->set_unique_id(_unique_id);
    column->set_name(_col_name);
    column->set_type(get_string_by_field_type(_type));
    column->set_is_key(_is_key);
    column->set_is_nullable(_is_nullable);
    column->set_is_auto_increment(_is_auto_increment);
    column->set_is_on_update_current_timestamp(_is_on_update_current_timestamp);
    if (_has_default_value) {
        column->set_default_value(_default_value);
    }
    if (_is_decimal) {
        column->set_precision(_precision);
        column->set_frac(_frac);
    }
    column->set_length(_length);
    column->set_index_length(_index_length);
    if (_is_bf_column) {
        column->set_is_bf_column(_is_bf_column);
    }
    if (!_aggregation_name.empty()) {
        column->set_aggregation(_aggregation_name);
    }
    column->set_result_is_nullable(_result_is_nullable);
    column->set_be_exec_version(_be_exec_version);
    column->set_visible(_visible);

    if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
        CHECK(_sub_columns.size() == 1)
                << "ARRAY type should has 1 children types, but got " << _sub_columns.size();
    }
    if (_type == FieldType::OLAP_FIELD_TYPE_MAP) {
        DCHECK(_sub_columns.size() == 2)
                << "MAP type should has 2 children types, but got " << _sub_columns.size();
        if (UNLIKELY(_sub_columns.size() != 2)) {
            LOG(WARNING) << "MAP type should has 2 children types, but got " << _sub_columns.size();
        }
    }

    for (size_t i = 0; i < _sub_columns.size(); i++) {
        ColumnPB* child = column->add_children_columns();
        _sub_columns[i]->to_schema_pb(child);
    }

    // set parts info
    if (has_path_info()) {
        // CHECK_GT(_parent_col_unique_id, 0);
        _column_path->to_protobuf(column->mutable_column_path_info(), _parent_col_unique_id);
        // Update unstable information for variant columns. Some of the fields in the tablet schema
        // are irrelevant for variant sub-columns, but retaining them may lead to an excessive growth
        // in the number of tablet schema cache entries.
        if (_type == FieldType::OLAP_FIELD_TYPE_STRING) {
            column->set_length(INT_MAX);
        }
        column->set_index_length(0);
    }
    column->set_variant_max_subcolumns_count(_variant.max_subcolumns_count);
    column->set_pattern_type(_pattern_type);
    column->set_variant_enable_typed_paths_to_sparse(_variant.enable_typed_paths_to_sparse);
    column->set_variant_max_sparse_column_statistics_size(
            _variant.max_sparse_column_statistics_size);
    column->set_variant_sparse_hash_shard_count(_variant.sparse_hash_shard_count);
    column->set_variant_enable_doc_mode(_variant.enable_doc_mode);
    column->set_variant_doc_materialization_min_rows(_variant.doc_materialization_min_rows);
    column->set_variant_doc_hash_shard_count(_variant.doc_hash_shard_count);
    column->set_variant_enable_nested_group(_variant.enable_nested_group);
}

void TabletColumn::add_sub_column(TabletColumn& sub_column) {
    _sub_columns.push_back(std::make_shared<TabletColumn>(sub_column));
    sub_column._parent_col_unique_id = this->_unique_id;
    _sub_column_count += 1;
}

bool TabletColumn::is_row_store_column() const {
    return _col_name == BeConsts::ROW_STORE_COL;
}

AggregateFunctionPtr TabletColumn::get_aggregate_function_union(DataTypePtr type,
                                                                int current_be_exec_version) const {
    const auto* state_type = assert_cast<const DataTypeAggState*>(type.get());
    BeExecVersionManager::check_function_compatibility(
            current_be_exec_version, _be_exec_version,
            state_type->get_nested_function()->get_name());
    return AggregateStateUnion::create(state_type->get_nested_function(), {type}, type);
}

AggregateFunctionPtr TabletColumn::get_aggregate_function(std::string suffix,
                                                          int current_be_exec_version) const {
    return get_aggregate_function(std::move(suffix), current_be_exec_version,
                                  DataTypeFactory::instance().create_data_type(*this));
}

AggregateFunctionPtr TabletColumn::get_aggregate_function(std::string suffix,
                                                          int current_be_exec_version,
                                                          DataTypePtr runtime_type) const {
    AggregateFunctionPtr function = nullptr;

    DORIS_CHECK(runtime_type != nullptr);
    if (runtime_type->get_primitive_type() == PrimitiveType::TYPE_AGG_STATE) {
        function = get_aggregate_function_union(runtime_type, current_be_exec_version);
    } else {
        std::string origin_name = TabletColumn::get_string_by_aggregation_type(_aggregation);
        std::string agg_name = origin_name + suffix;
        std::transform(agg_name.begin(), agg_name.end(), agg_name.begin(),
                       [](unsigned char c) { return std::tolower(c); });
        function = AggregateFunctionSimpleFactory::instance().get(
                agg_name, {runtime_type}, runtime_type, runtime_type->is_nullable(),
                BeExecVersionManager::get_newest_version());
        if (!function) {
            LOG(WARNING) << "get column aggregate function failed, aggregation_name=" << origin_name
                         << ", column_type=" << runtime_type->get_name();
        }
    }
    if (function) {
        function->set_version(_be_exec_version);
        return function;
    }
    return nullptr;
}

void TabletColumn::set_path_info(const PathInData& path) {
    _column_path = std::make_shared<PathInData>(path);
}

DataTypePtr TabletColumn::get_vec_type() const {
    return DataTypeFactory::instance().create_data_type(*this);
}

// escape '.' and '_'
std::string escape_for_path_name(const std::string& s) {
    std::string res;
    const char* pos = s.data();
    const char* end = pos + s.size();
    while (pos != end) {
        unsigned char c = *pos;
        if (c == '.' || c == '_') {
            res += '%';
            res += hex_digit_uppercase(c / 16);
            res += hex_digit_uppercase(c % 16);
        } else {
            res += c;
        }
        ++pos;
    }
    return res;
}

void TabletIndex::set_escaped_escaped_index_suffix_path(const std::string& path_name) {
    std::string escaped_path = escape_for_path_name(path_name);
    _escaped_index_suffix_path = escaped_path;
}

void TabletIndex::init_from_thrift(const TOlapTableIndex& index,
                                   const TabletSchema& tablet_schema) {
    _index_id = index.index_id;
    _index_name = index.index_name;
    // init col_unique_id in index at be side, since col_unique_id may be -1 at fe side
    // get column unique id by name
    std::vector<int32_t> col_unique_ids(index.columns.size());
    for (size_t i = 0; i < index.columns.size(); i++) {
        auto column_idx = tablet_schema.field_index(index.columns[i]);
        if (column_idx >= 0) {
            col_unique_ids[i] = tablet_schema.column(column_idx).unique_id();
        } else {
            // if column unique id not found by column name, find by column unique id
            // column unique id can not found means this column is a new column added by light schema change
            if (index.__isset.column_unique_ids && !index.column_unique_ids.empty() &&
                tablet_schema.has_column_unique_id(index.column_unique_ids[i])) {
                col_unique_ids[i] = index.column_unique_ids[i];
            } else {
                col_unique_ids[i] = -1;
            }
        }
    }
    _col_unique_ids = std::move(col_unique_ids);

    switch (index.index_type) {
    case TIndexType::BITMAP:
        _index_type = IndexType::BITMAP;
        break;
    case TIndexType::INVERTED:
        _index_type = IndexType::INVERTED;
        break;
    case TIndexType::ANN:
        _index_type = IndexType::ANN;
        break;
    case TIndexType::BLOOMFILTER:
        _index_type = IndexType::BLOOMFILTER;
        break;
    case TIndexType::NGRAM_BF:
        _index_type = IndexType::NGRAM_BF;
        break;
    }
    if (index.__isset.properties) {
        for (auto kv : index.properties) {
            _properties[kv.first] = kv.second;
        }
    }
}

void TabletIndex::init_from_thrift(const TOlapTableIndex& index,
                                   const std::vector<int32_t>& column_uids) {
    _index_id = index.index_id;
    _index_name = index.index_name;
    _col_unique_ids = column_uids;

    switch (index.index_type) {
    case TIndexType::BITMAP:
        _index_type = IndexType::BITMAP;
        break;
    case TIndexType::INVERTED:
        _index_type = IndexType::INVERTED;
        break;
    case TIndexType::ANN:
        _index_type = IndexType::ANN;
        break;
    case TIndexType::BLOOMFILTER:
        _index_type = IndexType::BLOOMFILTER;
        break;
    case TIndexType::NGRAM_BF:
        _index_type = IndexType::NGRAM_BF;
        break;
    }
    if (index.__isset.properties) {
        for (auto kv : index.properties) {
            _properties[kv.first] = kv.second;
        }
    }
}

void TabletIndex::init_from_pb(const TabletIndexPB& index) {
    _index_id = index.index_id();
    _index_name = index.index_name();
    _col_unique_ids.clear();
    for (auto col_unique_id : index.col_unique_id()) {
        _col_unique_ids.push_back(col_unique_id);
    }
    _index_type = index.index_type();
    for (const auto& kv : index.properties()) {
        _properties[kv.first] = kv.second;
    }
    _escaped_index_suffix_path = index.index_suffix_name();
}

void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
    index->set_index_id(_index_id);
    index->set_index_name(_index_name);
    index->clear_col_unique_id();
    for (auto col_unique_id : _col_unique_ids) {
        index->add_col_unique_id(col_unique_id);
    }
    index->set_index_type(_index_type);
    for (const auto& kv : _properties) {
        DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", {
            if (kv.first == INVERTED_INDEX_PARSER_LOWERCASE_KEY) {
                continue;
            }
        })
        (*index->mutable_properties())[kv.first] = kv.second;
    }
    index->set_index_suffix_name(_escaped_index_suffix_path);

    DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", { return; })

    // Only add lower_case=true default for built-in analyzers/parsers, NOT for custom analyzers
    // Custom analyzer: lower_case is determined by analyzer's internal token filter
    if (!_properties.empty() && !_properties.contains(INVERTED_INDEX_PARSER_LOWERCASE_KEY)) {
        bool has_parser = _properties.contains(INVERTED_INDEX_PARSER_KEY) ||
                          _properties.contains(INVERTED_INDEX_PARSER_KEY_ALIAS);
        std::string analyzer_name = get_analyzer_name_from_properties(_properties);
        bool is_builtin = analyzer_name.empty() ||
                          segment_v2::inverted_index::InvertedIndexAnalyzer::is_builtin_analyzer(
                                  analyzer_name);
        if (has_parser || is_builtin) {
            (*index->mutable_properties())[INVERTED_INDEX_PARSER_LOWERCASE_KEY] =
                    INVERTED_INDEX_PARSER_TRUE;
        }
    }
}

TabletSchema::TabletSchema() = default;

TabletSchema::~TabletSchema() {}

int64_t TabletSchema::get_metadata_size() const {
    return sizeof(TabletSchema);
}

void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
    if (column.is_key()) {
        _num_key_columns++;
    }
    if (column.is_nullable()) {
        _num_null_columns++;
    }
    if (column.is_variant_type()) {
        ++_num_variant_columns;
        if (!column.has_path_info()) {
            const std::string& col_name = column.name_lower_case();
            PathInData path(col_name);
            column.set_path_info(path);
        }
    }
    if (UNLIKELY(column.name() == DELETE_SIGN)) {
        _delete_sign_idx = _num_columns;
    } else if (UNLIKELY(column.name() == SEQUENCE_COL)) {
        _sequence_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == VERSION_COL)) {
        _version_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == SKIP_BITMAP_COL)) {
        _skip_bitmap_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == COMMIT_TSO_COL)) {
        _commit_tso_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == BINLOG_TSO_COL)) {
        _binlog_tso_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == BINLOG_LSN_COL)) {
        _binlog_lsn_col_idx = _num_columns;
    } else if (UNLIKELY(column.name() == BINLOG_OP_COL)) {
        _binlog_op_col_idx = _num_columns;
    } else if (UNLIKELY(column.name().starts_with(BeConsts::VIRTUAL_COLUMN_PREFIX))) {
        _vir_col_idx_to_unique_id[_num_columns] = column.unique_id();
    }
    _field_uniqueid_to_index[column.unique_id()] = _num_columns;
    _cols.push_back(std::make_shared<TabletColumn>(std::move(column)));
    // The dropped column may have same name with exsiting column, so that
    // not add to name to index map, only for uid to index map
    if (col_type == ColumnType::VARIANT || _cols.back()->is_variant_type() ||
        _cols.back()->is_extracted_column()) {
        _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
        _field_path_to_index[_cols.back()->path_info_ptr().get()] = _num_columns;
    } else if (col_type == ColumnType::NORMAL) {
        _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
    }
    _num_columns++;
    _num_virtual_columns = _vir_col_idx_to_unique_id.size();
    // generate column index mapping for seq map
    if (_seq_col_uid_to_value_cols_uid.contains(column.unique_id())) {
        const auto seq_idx = _field_uniqueid_to_index[column.unique_id()];
        if (!_seq_col_idx_to_value_cols_idx.contains(seq_idx)) {
            _seq_col_idx_to_value_cols_idx[seq_idx] = {};
        }
    }
    if (_value_col_uid_to_seq_col_uid.contains(column.unique_id())) {
        const auto seq_uid = _value_col_uid_to_seq_col_uid[column.unique_id()];
        if (_field_uniqueid_to_index.contains(seq_uid)) {
            bool all_uid_index_found = true;
            std::vector<int32_t> value_cols_index;
            for (const auto value_col_uid : _seq_col_uid_to_value_cols_uid[seq_uid]) {
                if (!_field_uniqueid_to_index.contains(value_col_uid)) {
                    all_uid_index_found = false;
                    break;
                }
                value_cols_index.push_back(_field_uniqueid_to_index[value_col_uid]);
            }
            if (all_uid_index_found) {
                const auto seq_idx = _field_uniqueid_to_index[seq_uid];
                for (const auto col_idx : value_cols_index) {
                    _seq_col_idx_to_value_cols_idx[seq_idx].push_back(col_idx);
                    _value_col_idx_to_seq_col_idx[col_idx] = seq_idx;
                }
                _value_col_idx_to_seq_col_idx[seq_idx] = seq_idx;
            }
        }
    }
}

void TabletSchema::append_index(TabletIndex&& index) {
    size_t index_pos = _indexes.size();
    _indexes.push_back(std::make_shared<TabletIndex>(index));
    for (int32_t id : _indexes.back()->col_unique_ids()) {
        if (auto field_pattern = _indexes.back()->field_pattern(); !field_pattern.empty()) {
            auto& pattern_to_index_map = _index_by_unique_id_with_pattern[id];
            pattern_to_index_map[field_pattern].emplace_back(_indexes.back());
        } else {
            IndexKey key = std::make_tuple(_indexes.back()->index_type(), id,
                                           _indexes.back()->get_index_suffix());
            _col_id_suffix_to_index[key].push_back(index_pos);
        }
    }
}

void TabletSchema::replace_column(size_t pos, TabletColumn new_col) {
    CHECK_LT(pos, num_columns()) << " outof range";
    _cols[pos] = std::make_shared<TabletColumn>(std::move(new_col));
}

void TabletSchema::clear_index() {
    _indexes.clear();
    _col_id_suffix_to_index.clear();
    _index_by_unique_id_with_pattern.clear();
}

void TabletSchema::remove_index(int64_t index_id) {
    std::vector<TabletIndexPtr> new_indexes;
    for (auto& index : _indexes) {
        if (index->index_id() != index_id) {
            new_indexes.emplace_back(std::move(index));
        }
    }
    _indexes = std::move(new_indexes);
    _col_id_suffix_to_index.clear();
    _index_by_unique_id_with_pattern.clear();
    for (size_t new_pos = 0; new_pos < _indexes.size(); ++new_pos) {
        const auto& index = _indexes[new_pos];
        for (int32_t col_uid : index->col_unique_ids()) {
            if (auto field_pattern = index->field_pattern(); !field_pattern.empty()) {
                auto& pattern_to_index_map = _index_by_unique_id_with_pattern[col_uid];
                pattern_to_index_map[field_pattern].emplace_back(index);
            } else {
                IndexKey key =
                        std::make_tuple(index->index_type(), col_uid, index->get_index_suffix());
                _col_id_suffix_to_index[key].push_back(new_pos);
            }
        }
    }
}

void TabletSchema::clear_columns() {
    _field_path_to_index.clear();
    _field_name_to_index.clear();
    _field_uniqueid_to_index.clear();
    _num_columns = 0;
    _num_variant_columns = 0;
    _num_null_columns = 0;
    _num_key_columns = 0;
    _seq_col_idx_to_value_cols_idx.clear();
    _value_col_idx_to_seq_col_idx.clear();
    _cols.clear();
}

void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns,
                                bool reuse_cache_column) {
    _keys_type = schema.keys_type();
    _num_columns = 0;
    _num_variant_columns = 0;
    _num_key_columns = 0;
    _num_null_columns = 0;
    _cols.clear();
    _indexes.clear();
    _index_by_unique_id_with_pattern.clear();
    _col_id_suffix_to_index.clear();
    _field_name_to_index.clear();
    _field_uniqueid_to_index.clear();
    _cluster_key_uids.clear();
    for (const auto& i : schema.cluster_key_uids()) {
        _cluster_key_uids.push_back(i);
    }
    for (auto& column_pb : schema.column()) {
        TabletColumnPtr column;
        if (reuse_cache_column) {
            auto pair = TabletColumnObjectPool::instance()->insert(
                    deterministic_string_serialize(column_pb));
            column = pair.second;
            // Release the handle quickly, because we use shared ptr to manage column.
            // It often core during tablet schema copy to another schema because handle's
            // reference count should be managed mannually.
            TabletColumnObjectPool::instance()->release(pair.first);
        } else {
            column = std::make_shared<TabletColumn>();
            column->init_from_pb(column_pb);
        }
        if (ignore_extracted_columns && column->is_extracted_column()) {
            continue;
        }
        if (column->is_key()) {
            _num_key_columns++;
        }
        if (column->is_nullable()) {
            _num_null_columns++;
        }
        if (column->is_variant_type()) {
            ++_num_variant_columns;
        }

        _cols.emplace_back(std::move(column));
        if (!_cols.back()->is_extracted_column()) {
            _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
            _field_uniqueid_to_index[_cols.back()->unique_id()] = _num_columns;
        }
        _num_columns++;
    }
    for (const auto& index_pb : schema.index()) {
        TabletIndexPtr index;
        if (reuse_cache_column) {
            auto pair = TabletColumnObjectPool::instance()->insert_index(
                    deterministic_string_serialize(index_pb));
            index = pair.second;
            //  Only need the value to be cached by the pool, release it quickly because the handle need
            // record reference count mannually, or it will core during tablet schema copy method.
            TabletColumnObjectPool::instance()->release(pair.first);
        } else {
            index = std::make_shared<TabletIndex>();
            index->init_from_pb(index_pb);
        }
        size_t index_pos = _indexes.size();
        _indexes.emplace_back(std::move(index));
        for (int32_t col_uid : _indexes.back()->col_unique_ids()) {
            if (auto field_pattern = _indexes.back()->field_pattern(); !field_pattern.empty()) {
                auto& pattern_to_index_map = _index_by_unique_id_with_pattern[col_uid];
                pattern_to_index_map[field_pattern].emplace_back(_indexes.back());
            } else {
                IndexKey key = std::make_tuple(_indexes.back()->index_type(), col_uid,
                                               _indexes.back()->get_index_suffix());
                _col_id_suffix_to_index[key].push_back(index_pos);
            }
        }
    }
    _num_short_key_columns = schema.num_short_key_columns();
    _num_rows_per_row_block = schema.num_rows_per_row_block();
    _compress_kind = schema.compress_kind();
    _next_column_unique_id = schema.next_column_unique_id();
    if (schema.has_bf_fpp()) {
        _has_bf_fpp = true;
        _bf_fpp = schema.bf_fpp();
    } else {
        _has_bf_fpp = false;
        _bf_fpp = BLOOM_FILTER_DEFAULT_FPP;
    }
    _is_in_memory = schema.is_in_memory();
    _disable_auto_compaction = schema.disable_auto_compaction();
    _store_row_column = schema.store_row_column();
    _skip_write_index_on_load = schema.skip_write_index_on_load();
    _delete_sign_idx = schema.delete_sign_idx();
    _sequence_col_idx = schema.sequence_col_idx();
    _version_col_idx = schema.version_col_idx();
    _skip_bitmap_col_idx = schema.skip_bitmap_col_idx();
    _commit_tso_col_idx = schema.commit_tso_col_idx();
    _binlog_tso_col_idx = schema.binlog_tso_col_idx();
    _binlog_lsn_col_idx = schema.binlog_lsn_col_idx();
    _binlog_op_col_idx = schema.binlog_op_col_idx();
    _sort_type = schema.sort_type();
    _sort_col_num = schema.sort_col_num();
    _compression_type = schema.compression_type();
    _row_store_page_size = schema.row_store_page_size();
    _storage_page_size = schema.storage_page_size();
    _storage_dict_page_size = schema.storage_dict_page_size();
    _schema_version = schema.schema_version();
    if (schema.has_seq_map()) {
        auto column_groups_pb = schema.seq_map();
        _seq_col_uid_to_value_cols_uid.clear();
        _value_col_uid_to_seq_col_uid.clear();
        _seq_col_idx_to_value_cols_idx.clear();
        _value_col_idx_to_seq_col_idx.clear();
        /*
         * ColumnGroupsPB is a list of cg_pb, and
         * ColumnGroupsPB do not have begin() or end() method.
         * we must use for(i=0;i<xx;i++) loop
         */
        for (int i = 0; i < column_groups_pb.cg_size(); i++) {
            ColumnGroupPB cg_pb = column_groups_pb.cg(i);
            uint32_t key_uid = cg_pb.sequence_column();
            auto found = _field_uniqueid_to_index.find(key_uid);
            DCHECK(found != _field_uniqueid_to_index.end())
                    << "could not find sequence col with unique id = " << key_uid
                    << " table_id=" << _table_id;
            int32_t seq_index = found->second;
            _seq_col_uid_to_value_cols_uid[key_uid] = {};
            _seq_col_idx_to_value_cols_idx[seq_index] = {};
            for (auto val_uid : cg_pb.columns_in_group()) {
                _seq_col_uid_to_value_cols_uid[key_uid].push_back(val_uid);
                found = _field_uniqueid_to_index.find(val_uid);
                DCHECK(found != _field_uniqueid_to_index.end())
                        << "could not find value col with unique id = " << key_uid
                        << " table_id=" << _table_id;
                int32_t val_index = found->second;
                _seq_col_idx_to_value_cols_idx[seq_index].push_back(val_index);
            }
        }

        if (!_seq_col_uid_to_value_cols_uid.empty()) {
            /*
                |** KEY **|        ** VALUE **     |
                ------------------------------------
                |** KEY **|  CDE is value| sequence|
                |----|----|----|----|----|----|----|
                A    B    C    D    E   S1   S2
                0    1    2    3    4    5    6
                for example: _seq_map is {5:{2,3}, 6:{4}}
                then, _value_to_seq = {2:5,3:5,5:5,4:6,6:6}
            */
            for (auto& [seq_uid, cols_uid] : _seq_col_uid_to_value_cols_uid) {
                for (auto col_uid : cols_uid) {
                    _value_col_uid_to_seq_col_uid[col_uid] = seq_uid;
                }
                _value_col_uid_to_seq_col_uid[seq_uid] = seq_uid;
            }

            for (auto& [seq_idx, value_cols_idx] : _seq_col_idx_to_value_cols_idx) {
                for (auto col_idx : value_cols_idx) {
                    _value_col_idx_to_seq_col_idx[col_idx] = seq_idx;
                }
                _value_col_idx_to_seq_col_idx[seq_idx] = seq_idx;
            }
        }
    }
    // Default to V1 inverted index storage format for backward compatibility if not specified in schema.
    if (!schema.has_inverted_index_storage_format()) {
        _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1;
    } else {
        _inverted_index_storage_format = schema.inverted_index_storage_format();
    }

    _row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
                                        schema.row_store_column_unique_ids().end());
    _deprecated_enable_variant_flatten_nested = schema.enable_variant_flatten_nested();
    if (schema.has_storage_format()) {
        _storage_format = schema.storage_format();
    } else if (schema.is_external_segment_column_meta_used() ||
               schema.integer_type_default_use_plain_encoding() ||
               schema.binary_plain_encoding_default_impl() ==
                       BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2) {
        // Old PB without storage_format: any of the three legacy V3-flavor flags implies V3.
        _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3;
    } else {
        _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
    }
    update_metadata_size();
}

void TabletSchema::copy_from(const TabletSchema& tablet_schema) {
    TabletSchemaPB tablet_schema_pb;
    tablet_schema.to_schema_pb(&tablet_schema_pb);
    init_from_pb(tablet_schema_pb);
    _table_id = tablet_schema.table_id();
    _path_set_info_map = tablet_schema._path_set_info_map;
}

void TabletSchema::shawdow_copy_without_columns(const TabletSchema& tablet_schema) {
    *this = tablet_schema;
    _field_path_to_index.clear();
    _field_name_to_index.clear();
    _field_uniqueid_to_index.clear();
    _num_columns = 0;
    _num_variant_columns = 0;
    _num_null_columns = 0;
    _num_key_columns = 0;
    _cols.clear();
    _delete_sign_idx = -1;
    _sequence_col_idx = -1;
    _version_col_idx = -1;
    _skip_bitmap_col_idx = -1;
    _commit_tso_col_idx = -1;
    _binlog_tso_col_idx = -1;
    _binlog_lsn_col_idx = -1;
    _binlog_op_col_idx = -1;
}

void TabletSchema::update_index_info_from(const TabletSchema& tablet_schema) {
    for (auto& col : _cols) {
        if (col->unique_id() < 0) {
            continue;
        }
        const auto iter = tablet_schema._field_uniqueid_to_index.find(col->unique_id());
        if (iter == tablet_schema._field_uniqueid_to_index.end()) {
            continue;
        }
        auto col_idx = iter->second;
        if (col_idx < 0 || col_idx >= tablet_schema._cols.size()) {
            continue;
        }
        col->set_is_bf_column(tablet_schema._cols[col_idx]->is_bf_column());
    }
}

std::string TabletSchema::to_key() const {
    TabletSchemaPB pb;
    to_schema_pb(&pb);
    return TabletSchema::deterministic_string_serialize(pb);
}

void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version,
                                               const OlapTableIndexSchema* index,
                                               const TabletSchema& ori_tablet_schema) {
    // copy from ori_tablet_schema
    _keys_type = ori_tablet_schema.keys_type();
    _num_short_key_columns = ori_tablet_schema.num_short_key_columns();
    _num_rows_per_row_block = ori_tablet_schema.num_rows_per_row_block();
    _compress_kind = ori_tablet_schema.compress_kind();

    // todo(yixiu): unique_id
    _next_column_unique_id = ori_tablet_schema.next_column_unique_id();
    _is_in_memory = ori_tablet_schema.is_in_memory();
    _disable_auto_compaction = ori_tablet_schema.disable_auto_compaction();
    _skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load();
    _sort_type = ori_tablet_schema.sort_type();
    _sort_col_num = ori_tablet_schema.sort_col_num();
    _row_store_page_size = ori_tablet_schema.row_store_page_size();
    _storage_page_size = ori_tablet_schema.storage_page_size();
    _storage_dict_page_size = ori_tablet_schema.storage_dict_page_size();
    _deprecated_enable_variant_flatten_nested =
            ori_tablet_schema.deprecated_variant_flatten_nested();

    // copy from table_schema_param
    _schema_version = version;
    _num_columns = 0;
    _num_variant_columns = 0;
    _num_key_columns = 0;
    _num_null_columns = 0;
    bool has_bf_columns = false;
    _cols.clear();
    _indexes.clear();
    _col_id_suffix_to_index.clear();
    _index_by_unique_id_with_pattern.clear();
    _field_name_to_index.clear();
    _field_uniqueid_to_index.clear();
    _delete_sign_idx = -1;
    _sequence_col_idx = -1;
    _version_col_idx = -1;
    _skip_bitmap_col_idx = -1;
    _commit_tso_col_idx = -1;
    _binlog_tso_col_idx = -1;
    _binlog_lsn_col_idx = -1;
    _binlog_op_col_idx = -1;
    _cluster_key_uids.clear();
    for (const auto& i : ori_tablet_schema._cluster_key_uids) {
        _cluster_key_uids.push_back(i);
    }
    for (auto& column : index->columns) {
        if (column->is_key()) {
            _num_key_columns++;
        }
        if (column->is_nullable()) {
            _num_null_columns++;
        }
        if (column->is_bf_column()) {
            has_bf_columns = true;
        }
        if (column->is_variant_type()) {
            ++_num_variant_columns;
        }
        if (UNLIKELY(column->name() == DELETE_SIGN)) {
            _delete_sign_idx = _num_columns;
        } else if (UNLIKELY(column->name() == SEQUENCE_COL)) {
            _sequence_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == VERSION_COL)) {
            _version_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == SKIP_BITMAP_COL)) {
            _skip_bitmap_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == COMMIT_TSO_COL)) {
            _commit_tso_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == BINLOG_TSO_COL)) {
            _binlog_tso_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == BINLOG_LSN_COL)) {
            _binlog_lsn_col_idx = _num_columns;
        } else if (UNLIKELY(column->name() == BINLOG_OP_COL)) {
            _binlog_op_col_idx = _num_columns;
        }
        // Reuse TabletColumn object from pool to reduce memory consumption
        TabletColumnPtr new_column;
        ColumnPB column_pb;
        column->to_schema_pb(&column_pb);
        auto pair = TabletColumnObjectPool::instance()->insert(
                deterministic_string_serialize(column_pb));
        new_column = pair.second;
        // Release the handle quickly, because we use shared ptr to manage column
        TabletColumnObjectPool::instance()->release(pair.first);
        _cols.emplace_back(std::move(new_column));
        _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
        _field_uniqueid_to_index[_cols.back()->unique_id()] = _num_columns;
        _num_columns++;
    }

    for (const auto& i : index->indexes) {
        size_t index_pos = _indexes.size();
        // Reuse TabletIndex object from pool to reduce memory consumption
        TabletIndexPtr new_index;
        TabletIndexPB index_pb;
        i->to_schema_pb(&index_pb);
        auto pair = TabletColumnObjectPool::instance()->insert_index(
                deterministic_string_serialize(index_pb));
        new_index = pair.second;
        // Release the handle quickly, because we use shared ptr to manage index
        TabletColumnObjectPool::instance()->release(pair.first);
        _indexes.emplace_back(std::move(new_index));
        for (int32_t col_uid : _indexes.back()->col_unique_ids()) {
            if (auto field_pattern = _indexes.back()->field_pattern(); !field_pattern.empty()) {
                auto& pattern_to_index_map = _index_by_unique_id_with_pattern[col_uid];
                pattern_to_index_map[field_pattern].emplace_back(_indexes.back());
            } else {
                IndexKey key = std::make_tuple(_indexes.back()->index_type(), col_uid,
                                               _indexes.back()->get_index_suffix());
                _col_id_suffix_to_index[key].push_back(index_pos);
            }
        }
    }

    if (has_bf_columns) {
        _has_bf_fpp = true;
        _bf_fpp = ori_tablet_schema.bloom_filter_fpp();
    } else {
        _has_bf_fpp = false;
        _bf_fpp = BLOOM_FILTER_DEFAULT_FPP;
    }
}

void TabletSchema::merge_dropped_columns(const TabletSchema& src_schema) {
    // If they are the same tablet schema object, then just return
    if (this == &src_schema) {
        return;
    }
    for (const auto& src_col : src_schema.columns()) {
        if (_field_uniqueid_to_index.find(src_col->unique_id()) == _field_uniqueid_to_index.end()) {
            CHECK(!src_col->is_key())
                    << src_col->name() << " is key column, should not be dropped.";
            ColumnPB src_col_pb;
            // There are some pointer in tablet column, not sure the reference relation, so
            // that deep copy it.
            src_col->to_schema_pb(&src_col_pb);
            TabletColumn new_col(src_col_pb);
            append_column(new_col, TabletSchema::ColumnType::DROPPED);
        }
    }
}

TabletSchemaSPtr TabletSchema::copy_without_variant_extracted_columns() {
    TabletSchemaSPtr copy = std::make_shared<TabletSchema>();
    copy->shawdow_copy_without_columns(*this);
    for (auto& col : this->columns()) {
        if (col->is_extracted_column()) {
            continue;
        }
        copy->append_column(*col);
    }
    return copy;
}

// Dropped column is in _field_uniqueid_to_index but not in _field_name_to_index
// Could refer to append_column method
bool TabletSchema::is_dropped_column(const TabletColumn& col) const {
    CHECK(_field_uniqueid_to_index.find(col.unique_id()) != _field_uniqueid_to_index.end())
            << "could not find col with unique id = " << col.unique_id()
            << " and name = " << col.name() << " table_id=" << _table_id;
    auto it = _field_name_to_index.find(StringRef {col.name()});
    return it == _field_name_to_index.end() || _cols[it->second]->unique_id() != col.unique_id();
}

void TabletSchema::copy_extracted_columns(const TabletSchema& src_schema) {
    std::unordered_set<int32_t> variant_columns;
    for (const auto& col : columns()) {
        if (col->is_variant_type()) {
            variant_columns.insert(col->unique_id());
        }
    }
    for (const TabletColumnPtr& col : src_schema.columns()) {
        if (col->is_extracted_column() && variant_columns.contains(col->parent_unique_id())) {
            ColumnPB col_pb;
            col->to_schema_pb(&col_pb);
            TabletColumn new_col(col_pb);
            append_column(new_col, ColumnType::VARIANT);
        }
    }
}

void TabletSchema::reserve_extracted_columns() {
    for (auto it = _cols.begin(); it != _cols.end();) {
        if (!(*it)->is_extracted_column()) {
            it = _cols.erase(it);
        } else {
            ++it;
        }
    }
}

void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
    for (const auto& i : _cluster_key_uids) {
        tablet_schema_pb->add_cluster_key_uids(i);
    }
    tablet_schema_pb->set_keys_type(_keys_type);
    for (const auto& col : _cols) {
        ColumnPB* column = tablet_schema_pb->add_column();
        col->to_schema_pb(column);
    }
    for (const auto& index : _indexes) {
        auto* index_pb = tablet_schema_pb->add_index();
        index->to_schema_pb(index_pb);
    }
    tablet_schema_pb->set_num_short_key_columns(cast_set<int32_t>(_num_short_key_columns));
    tablet_schema_pb->set_num_rows_per_row_block(cast_set<int32_t>(_num_rows_per_row_block));
    tablet_schema_pb->set_compress_kind(_compress_kind);
    if (_has_bf_fpp) {
        tablet_schema_pb->set_bf_fpp(_bf_fpp);
    }
    tablet_schema_pb->set_next_column_unique_id(cast_set<uint32_t>(_next_column_unique_id));
    tablet_schema_pb->set_is_in_memory(_is_in_memory);
    tablet_schema_pb->set_disable_auto_compaction(_disable_auto_compaction);
    tablet_schema_pb->set_store_row_column(_store_row_column);
    tablet_schema_pb->set_skip_write_index_on_load(_skip_write_index_on_load);
    tablet_schema_pb->set_delete_sign_idx(_delete_sign_idx);
    tablet_schema_pb->set_sequence_col_idx(_sequence_col_idx);
    tablet_schema_pb->set_sort_type(_sort_type);
    tablet_schema_pb->set_sort_col_num(cast_set<int32_t>(_sort_col_num));
    tablet_schema_pb->set_schema_version(_schema_version);
    tablet_schema_pb->set_compression_type(_compression_type);
    tablet_schema_pb->set_row_store_page_size(_row_store_page_size);
    tablet_schema_pb->set_storage_page_size(_storage_page_size);
    tablet_schema_pb->set_storage_dict_page_size(_storage_dict_page_size);
    tablet_schema_pb->set_version_col_idx(_version_col_idx);
    tablet_schema_pb->set_skip_bitmap_col_idx(_skip_bitmap_col_idx);
    tablet_schema_pb->set_commit_tso_col_idx(_commit_tso_col_idx);
    tablet_schema_pb->set_binlog_tso_col_idx(_binlog_tso_col_idx);
    tablet_schema_pb->set_binlog_lsn_col_idx(_binlog_lsn_col_idx);
    tablet_schema_pb->set_binlog_op_col_idx(_binlog_op_col_idx);
    tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
    tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign(
            _row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end());
    tablet_schema_pb->set_enable_variant_flatten_nested(_deprecated_enable_variant_flatten_nested);
    tablet_schema_pb->set_storage_format(_storage_format);
    // Backward downgrade safety: if a new BE rewrites tablet_meta.json carrying only
    // storage_format and the deployment is then rolled back to an old BE, the old BE
    // does not know the new field and would default-derive V2 for a V3 tablet, causing
    // it to write V2-encoded segments into a V3 tablet. Redundantly emit the three
    // legacy V3-flavor flags so old BEs can recover the format via the prior "any of
    // these implies V3" rule. ~3 bytes per schema PB; only paid for V3 tablets.
    if (_storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) {
        tablet_schema_pb->set_is_external_segment_column_meta_used(true);
        tablet_schema_pb->set_integer_type_default_use_plain_encoding(true);
        tablet_schema_pb->set_binary_plain_encoding_default_impl(
                BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2);
    }
    auto column_groups_pb = tablet_schema_pb->mutable_seq_map();
    for (const auto& it : _seq_col_uid_to_value_cols_uid) {
        uint32_t key = it.first;
        ColumnGroupPB* cg_pb = column_groups_pb->add_cg(); // ColumnGroupPB {key: {v1, v2, v3}}
        cg_pb->set_sequence_column(key);
        for (auto v : it.second) {
            cg_pb->add_columns_in_group(v);
        }
    }
}

size_t TabletSchema::row_size() const {
    size_t size = 0;
    for (const auto& column : _cols) {
        size += column->length();
    }
    size += (_num_columns + 7) / 8;

    return size;
}

int32_t TabletSchema::field_index(const std::string& field_name) const {
    const auto& found = _field_name_to_index.find(StringRef(field_name));
    return (found == _field_name_to_index.end()) ? -1 : found->second;
}

int32_t TabletSchema::field_index(const PathInData& path) const {
    const auto& found = _field_path_to_index.find(PathInDataRef(&path));
    return (found == _field_path_to_index.end()) ? -1 : found->second;
}

int32_t TabletSchema::field_index(int32_t col_unique_id) const {
    const auto& found = _field_uniqueid_to_index.find(col_unique_id);
    return (found == _field_uniqueid_to_index.end()) ? -1 : found->second;
}

const std::vector<TabletColumnPtr>& TabletSchema::columns() const {
    return _cols;
}

const TabletColumn& TabletSchema::column(size_t ordinal) const {
    DCHECK(ordinal < _num_columns) << "ordinal:" << ordinal << ", _num_columns:" << _num_columns;
    return *_cols[ordinal];
}

const TabletColumn& TabletSchema::column_by_uid(int32_t col_unique_id) const {
    return *_cols.at(_field_uniqueid_to_index.at(col_unique_id));
}

TabletColumn& TabletSchema::mutable_column_by_uid(int32_t col_unique_id) {
    return *_cols.at(_field_uniqueid_to_index.at(col_unique_id));
}

TabletColumn& TabletSchema::mutable_column(size_t ordinal) {
    return *_cols.at(ordinal);
}

void TabletSchema::update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>& tindexes) {
    std::vector<TabletIndexPtr> indexes;
    for (const auto& tindex : tindexes) {
        TabletIndex index;
        index.init_from_thrift(tindex, *this);
        indexes.emplace_back(std::make_shared<TabletIndex>(std::move(index)));
    }
    _indexes = std::move(indexes);
    _col_id_suffix_to_index.clear();
    _index_by_unique_id_with_pattern.clear();
    size_t index_pos = 0;
    for (auto& index : _indexes) {
        for (int32_t col_uid : index->col_unique_ids()) {
            if (auto field_pattern = index->field_pattern(); !field_pattern.empty()) {
                auto& pattern_to_index_map = _index_by_unique_id_with_pattern[col_uid];
                pattern_to_index_map[field_pattern].emplace_back(index);
            } else {
                IndexKey key =
                        std::make_tuple(index->index_type(), col_uid, index->get_index_suffix());
                _col_id_suffix_to_index[key].push_back(index_pos);
            }
        }
        index_pos++;
    }
}

bool TabletSchema::exist_column(const std::string& field_name) const {
    return _field_name_to_index.contains(StringRef {field_name});
}

bool TabletSchema::has_column_unique_id(int32_t col_unique_id) const {
    return _field_uniqueid_to_index.contains(col_unique_id);
}

Status TabletSchema::have_column(const std::string& field_name) const {
    if (!_field_name_to_index.contains(StringRef(field_name))) {
        return Status::Error<ErrorCode::INTERNAL_ERROR>(
                "Not found field_name, field_name:{}, schema:{}", field_name,
                get_all_field_names());
    }
    return Status::OK();
}

Result<const TabletColumn*> TabletSchema::column(const std::string& field_name) const {
    auto it = _field_name_to_index.find(StringRef {field_name});
    if (it == _field_name_to_index.end()) {
        DCHECK(false) << "field_name=" << field_name << ", table_id=" << _table_id
                      << ", field_name_to_index=" << get_all_field_names();
        return ResultError(
                Status::InternalError("column not found, name={}, table_id={}, schema_version={}",
                                      field_name, _table_id, _schema_version));
    }
    return _cols[it->second].get();
}

void TabletSchema::update_tablet_columns(const TabletSchema& tablet_schema,
                                         const std::vector<TColumn>& t_columns) {
    copy_from(tablet_schema);
    if (!t_columns.empty() && t_columns[0].col_unique_id >= 0) {
        clear_columns();
        for (const auto& column : t_columns) {
            append_column(TabletColumn(column));
        }
    }
}

bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id) const {
    for (size_t i = 0; i < _indexes.size(); i++) {
        if ((_indexes[i]->index_type() == IndexType::INVERTED ||
             _indexes[i]->index_type() == IndexType::ANN) &&
            _indexes[i]->index_id() == index_id) {
            return true;
        }
    }
    return false;
}

std::vector<const TabletIndex*> TabletSchema::inverted_indexs(
        int32_t col_unique_id, const std::string& suffix_path) const {
    std::vector<const TabletIndex*> result;
    const std::string escaped_suffix = escape_for_path_name(suffix_path);
    auto it = _col_id_suffix_to_index.find(
            std::make_tuple(IndexType::INVERTED, col_unique_id, escaped_suffix));
    if (it != _col_id_suffix_to_index.end()) {
        for (size_t pos : it->second) {
            if (pos < _indexes.size()) {
                result.push_back(_indexes[pos].get());
            }
        }
    }
    return result;
}

std::vector<TabletIndexPtr> TabletSchema::inverted_index_by_field_pattern(
        int32_t col_unique_id, const std::string& field_pattern) const {
    auto id_to_pattern_map = _index_by_unique_id_with_pattern.find(col_unique_id);
    if (id_to_pattern_map == _index_by_unique_id_with_pattern.end()) {
        return {};
    }
    auto pattern_to_index_map = id_to_pattern_map->second.find(field_pattern);
    if (pattern_to_index_map == id_to_pattern_map->second.end()) {
        return {};
    }
    return pattern_to_index_map->second;
}

std::vector<const TabletIndex*> TabletSchema::inverted_indexs(const TabletColumn& col) const {
    // Some columns(Float, Double, JSONB ...) from the variant do not support inverted index
    if (!segment_v2::IndexColumnWriter::check_support_inverted_index(col)) {
        return {};
    }

    // TODO use more efficient impl
    // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants
    int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id();
    std::vector<const TabletIndex*> result;
    if (result = inverted_indexs(col_unique_id, escape_for_path_name(col.suffix_path()));
        !result.empty()) {
        return result;
    }
    // variant's typed column has it's own index
    else if (col.is_extracted_column() && col.path_info_ptr()->get_is_typed()) {
        std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path();
        if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) {
            return result;
        }
        const auto& path_set_info = _path_set_info_map.at(col_unique_id);
        if (path_set_info.typed_path_set.find(relative_path) ==
            path_set_info.typed_path_set.end()) {
            return result;
        }
        for (const auto& index : path_set_info.typed_path_set.at(relative_path).indexes) {
            result.push_back(index.get());
        }
        return result;
    }
    // variant's subcolumns has it's own index
    else if (col.is_extracted_column()) {
        std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path();
        if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) {
            return result;
        }
        const auto& path_set_info = _path_set_info_map.at(col_unique_id);
        if (path_set_info.subcolumn_indexes.find(relative_path) ==
            path_set_info.subcolumn_indexes.end()) {
            return result;
        }
        for (const auto& index : path_set_info.subcolumn_indexes.at(relative_path)) {
            result.push_back(index.get());
        }
    }
    return result;
}

const TabletIndex* TabletSchema::ann_index(int32_t col_unique_id,
                                           const std::string& suffix_path) const {
    for (size_t i = 0; i < _indexes.size(); i++) {
        if (_indexes[i]->index_type() == IndexType::ANN) {
            for (int32_t id : _indexes[i]->col_unique_ids()) {
                if (id == col_unique_id &&
                    _indexes[i]->get_index_suffix() == escape_for_path_name(suffix_path)) {
                    return _indexes[i].get();
                }
            }
        }
    }
    return nullptr;
}

const TabletIndex* TabletSchema::ann_index(const TabletColumn& col) const {
    if (!segment_v2::IndexColumnWriter::check_support_ann_index(col)) {
        return nullptr;
    }
    // TODO use more efficient impl
    // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants
    int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id();
    return ann_index(col_unique_id, escape_for_path_name(col.suffix_path()));
}

bool TabletSchema::has_ngram_bf_index(int32_t col_unique_id) const {
    IndexKey index_key(IndexType::NGRAM_BF, col_unique_id, "");
    auto it = _col_id_suffix_to_index.find(index_key);
    return it != _col_id_suffix_to_index.end();
}

const TabletIndex* TabletSchema::get_ngram_bf_index(int32_t col_unique_id) const {
    // Get the ngram bf index for the given column unique id
    IndexKey index_key(IndexType::NGRAM_BF, col_unique_id, "");
    auto it = _col_id_suffix_to_index.find(index_key);
    if (it != _col_id_suffix_to_index.end()) {
        if (!it->second.empty() && it->second[0] < _indexes.size()) {
            return _indexes[it->second[0]].get();
        }
    }
    return nullptr;
}

const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType index_type,
                                           const std::string& suffix_path) const {
    IndexKey index_key(index_type, col_unique_id, suffix_path);
    auto it = _col_id_suffix_to_index.find(index_key);
    if (it != _col_id_suffix_to_index.end()) {
        if (!it->second.empty() && it->second[0] < _indexes.size()) {
            return _indexes[it->second[0]].get();
        }
    }
    return nullptr;
}

Block TabletSchema::create_block(
        const std::vector<uint32_t>& return_columns,
        const std::unordered_set<uint32_t>* tablet_columns_need_convert_null) const {
    Block block;
    for (int i = 0; i < return_columns.size(); ++i) {
        const ColumnId cid = return_columns[i];
        const auto& col = *_cols[cid];
        bool is_nullable = (tablet_columns_need_convert_null != nullptr &&
                            tablet_columns_need_convert_null->find(cid) !=
                                    tablet_columns_need_convert_null->end());
        auto data_type = DataTypeFactory::instance().create_data_type(col, is_nullable);
        if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT ||
            col.type() == FieldType::OLAP_FIELD_TYPE_MAP ||
            col.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
            if (_pruned_columns_data_type.contains(col.unique_id())) {
                data_type = _pruned_columns_data_type.at(col.unique_id());
            }
        }

        if (_vir_col_idx_to_unique_id.contains(cid)) {
            block.insert({ColumnNothing::create(0), data_type, col.name()});
            VLOG_DEBUG << fmt::format(
                    "Create block from tablet schema, column cid {} is virtual column, col_name: "
                    "{}, col_unique_id: {}, type {}",
                    cid, col.name(), col.unique_id(), data_type->get_name());
        } else {
            block.insert({data_type->create_column(), data_type, col.name()});
        }
    }
    return block;
}

Block TabletSchema::create_block() const {
    Block block;
    for (const auto& col : _cols) {
        if (is_dropped_column(*col)) {
            continue;
        }

        auto data_type = DataTypeFactory::instance().create_data_type(*col);
        if (col->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
            if (_pruned_columns_data_type.contains(col->unique_id())) {
                data_type = _pruned_columns_data_type.at(col->unique_id());
            }
        }
        block.insert({data_type->create_column(), data_type, col->name()});
    }
    return block;
}

Block TabletSchema::create_block_by_cids(const std::vector<uint32_t>& cids) const {
    Block block;
    for (const auto& cid : cids) {
        const auto& col = *_cols[cid];
        auto data_type = DataTypeFactory::instance().create_data_type(col);
        if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
            if (_pruned_columns_data_type.contains(col.unique_id())) {
                data_type = _pruned_columns_data_type.at(col.unique_id());
            }
        }
        block.insert({data_type->create_column(), data_type, col.name()});
    }
    return block;
}

bool operator==(const TabletColumn& a, const TabletColumn& b) {
    if (a._unique_id != b._unique_id) return false;
    if (a._col_name != b._col_name) return false;
    if (a._type != b._type) return false;
    if (a._is_key != b._is_key) return false;
    if (a._aggregation != b._aggregation) return false;
    if (a._is_nullable != b._is_nullable) return false;
    if (a._has_default_value != b._has_default_value) return false;
    if (a._has_default_value) {
        if (a._default_value != b._default_value) return false;
    }
    if (a._is_decimal != b._is_decimal) return false;
    if (a._is_decimal) {
        if (a._precision != b._precision) return false;
        if (a._frac != b._frac) return false;
    }
    if (a._length != b._length) return false;
    if (a._index_length != b._index_length) return false;
    if (a._is_bf_column != b._is_bf_column) return false;
    if (a._column_path == nullptr && a._column_path != nullptr) return false;
    if (b._column_path == nullptr && a._column_path != nullptr) return false;
    if (b._column_path != nullptr && a._column_path != nullptr &&
        *a._column_path != *b._column_path)
        return false;
    return true;
}

bool operator!=(const TabletColumn& a, const TabletColumn& b) {
    return !(a == b);
}

bool operator==(const TabletSchema& a, const TabletSchema& b) {
    if (a._keys_type != b._keys_type) return false;
    if (a._cols.size() != b._cols.size()) return false;
    for (int i = 0; i < a._cols.size(); ++i) {
        if (*a._cols[i] != *b._cols[i]) return false;
    }
    if (a._num_columns != b._num_columns) return false;
    if (a._num_key_columns != b._num_key_columns) return false;
    if (a._num_null_columns != b._num_null_columns) return false;
    if (a._num_short_key_columns != b._num_short_key_columns) return false;
    if (a._num_rows_per_row_block != b._num_rows_per_row_block) return false;
    if (a._compress_kind != b._compress_kind) return false;
    if (a._next_column_unique_id != b._next_column_unique_id) return false;
    if (a._has_bf_fpp != b._has_bf_fpp) return false;
    if (a._has_bf_fpp) {
        if (std::abs(a._bf_fpp - b._bf_fpp) > 1e-6) return false;
    }
    if (a._is_in_memory != b._is_in_memory) return false;
    if (a._delete_sign_idx != b._delete_sign_idx) return false;
    if (a._sequence_col_idx != b._sequence_col_idx) return false;
    if (a._version_col_idx != b._version_col_idx) return false;
    if (a._skip_bitmap_col_idx != b._skip_bitmap_col_idx) return false;
    if (a._commit_tso_col_idx != b._commit_tso_col_idx) return false;
    if (a._binlog_tso_col_idx != b._binlog_tso_col_idx) return false;
    if (a._binlog_lsn_col_idx != b._binlog_lsn_col_idx) return false;
    if (a._binlog_op_col_idx != b._binlog_op_col_idx) return false;
    if (a._disable_auto_compaction != b._disable_auto_compaction) return false;
    if (a._store_row_column != b._store_row_column) return false;
    if (a._row_store_page_size != b._row_store_page_size) return false;
    if (a._storage_page_size != b._storage_page_size) return false;
    if (a._storage_dict_page_size != b._storage_dict_page_size) return false;
    if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false;
    if (a._deprecated_enable_variant_flatten_nested !=
        b._deprecated_enable_variant_flatten_nested) {
        return false;
    }
    if (a._storage_format != b._storage_format) return false;
    return true;
}

bool operator!=(const TabletSchema& a, const TabletSchema& b) {
    return !(a == b);
}
} // namespace doris

Coverage Report

Created: 2026-08-14 07:23