be/src/storage/tablet/tablet_schema.h

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/olap_common.pb.h>
#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>
#include <parallel_hashmap/phmap.h>

#include <algorithm>
#include <cstdint>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "common/consts.h"
#include "common/status.h"
#include "core/data_type/define_primitive_type.h"
#include "core/string_ref.h"
#include "core/types.h"
#include "exec/common/string_utils/string_utils.h"
#include "exprs/aggregate/aggregate_function.h"
#include "runtime/descriptors.h"
#include "runtime/memory/lru_cache_policy.h"
#include "storage/index/inverted/inverted_index_parser.h"
#include "storage/metadata_adder.h"
#include "storage/olap_common.h"
#include "storage/segment/options.h"
#include "util/debug_points.h"
#include "util/json/path_in_data.h"
#include "util/string_parser.hpp"
#include "util/string_util.h"

namespace doris {
class Block;
class PathInData;
class IDataType;

struct OlapTableIndexSchema;
class TColumn;
class TOlapTableIndex;
class TabletColumn;

using TabletColumnPtr = std::shared_ptr<TabletColumn>;

class TabletColumn : public MetadataAdder<TabletColumn> {
public:
    struct VariantParams {
        int32_t max_subcolumns_count = 0;
        bool enable_typed_paths_to_sparse = false;
        int32_t max_sparse_column_statistics_size =
                BeConsts::DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATS_SIZE;
        // default to 0, no shard
        int32_t sparse_hash_shard_count = 0;

        bool enable_doc_mode = false;
        int64_t doc_materialization_min_rows = 0;
        int32_t doc_hash_shard_count = 64;

        bool enable_nested_group = false;
    };

    TabletColumn();
    TabletColumn(const ColumnPB& column);
    TabletColumn(const TColumn& column);
    TabletColumn(FieldAggregationMethod agg, FieldType type);
    TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable);
    TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable,
                 int32_t unique_id, size_t length);

#ifdef BE_TEST
    virtual ~TabletColumn() = default;
#endif

    void init_from_pb(const ColumnPB& column);
    void init_from_thrift(const TColumn& column);
    void to_schema_pb(ColumnPB* column) const;

    int32_t unique_id() const { return _unique_id; }
    void set_unique_id(int32_t id) { _unique_id = id; }
    const std::string& name() const { return _col_name; }
    const std::string& name_lower_case() const { return _col_name_lower_case; }
    void set_name(std::string col_name) {
        _col_name = col_name;
        _col_name_lower_case = to_lower(_col_name);
    }
    MOCK_FUNCTION FieldType type() const { return _type; }
    void set_type(FieldType type) { _type = type; }
    bool is_key() const { return _is_key; }
    bool is_nullable() const { return _is_nullable; }
    bool is_auto_increment() const { return _is_auto_increment; }
    bool is_seqeunce_col() const { return _col_name == SEQUENCE_COL; }
    bool is_on_update_current_timestamp() const { return _is_on_update_current_timestamp; }
    bool is_variant_type() const { return _type == FieldType::OLAP_FIELD_TYPE_VARIANT; }
    bool is_bf_column() const { return _is_bf_column; }
    bool is_array_type() const { return _type == FieldType::OLAP_FIELD_TYPE_ARRAY; }
    bool is_agg_state_type() const { return _type == FieldType::OLAP_FIELD_TYPE_AGG_STATE; }
    bool is_jsonb_type() const { return _type == FieldType::OLAP_FIELD_TYPE_JSONB; }
    bool is_length_variable_type() const {
        return _type == FieldType::OLAP_FIELD_TYPE_CHAR ||
               _type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
               _type == FieldType::OLAP_FIELD_TYPE_STRING ||
               _type == FieldType::OLAP_FIELD_TYPE_HLL ||
               _type == FieldType::OLAP_FIELD_TYPE_BITMAP ||
               _type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
               _type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
    }
    bool has_default_value() const { return _has_default_value; }
    std::string default_value() const { return _default_value; }
    bool has_default_value_expr() const { return _has_default_value_expr; }
    const std::string& default_value_expr() const { return _default_value_expr; }
    int32_t length() const { return _length; }
    void set_length(int32_t length) { _length = length; }
    void set_default_value(const std::string& default_value) {
        _default_value = default_value;
        _has_default_value = true;
    }
    void set_default_value_expr(const std::string& default_value_expr) {
        _default_value_expr = default_value_expr;
        _has_default_value_expr = true;
    }
    int32_t index_length() const { return _index_length; }
    void set_index_length(int32_t index_length) { _index_length = index_length; }
    void set_is_key(bool is_key) { _is_key = is_key; }
    void set_is_nullable(bool is_nullable) { _is_nullable = is_nullable; }
    void set_is_auto_increment(bool is_auto_increment) { _is_auto_increment = is_auto_increment; }
    void set_is_on_update_current_timestamp(bool is_on_update_current_timestamp) {
        _is_on_update_current_timestamp = is_on_update_current_timestamp;
    }
    void set_path_info(const PathInData& path);
    FieldAggregationMethod aggregation() const { return _aggregation; }
    AggregateFunctionPtr get_aggregate_function_union(DataTypePtr type,
                                                      int current_be_exec_version) const;
    AggregateFunctionPtr get_aggregate_function(std::string suffix,
                                                int current_be_exec_version) const;
    AggregateFunctionPtr get_aggregate_function(std::string suffix, int current_be_exec_version,
                                                DataTypePtr runtime_type) const;
    int precision() const { return _precision; }
    int frac() const { return _frac; }
    inline bool visible() const { return _visible; }
    bool has_char_type() const;

    void set_aggregation_method(FieldAggregationMethod agg) {
        _aggregation = agg;
        _aggregation_name = get_string_by_aggregation_type(agg);
    }

    /**
     * Add a sub column.
     */
    void add_sub_column(TabletColumn& sub_column);

    uint32_t get_subtype_count() const { return _sub_column_count; }
    MOCK_FUNCTION const TabletColumn& get_sub_column(uint64_t i) const { return *_sub_columns[i]; }
    const std::vector<TabletColumnPtr>& get_sub_columns() const { return _sub_columns; }

    friend bool operator==(const TabletColumn& a, const TabletColumn& b);
    friend bool operator!=(const TabletColumn& a, const TabletColumn& b);

    static std::string get_string_by_field_type(FieldType type);
    static std::string get_string_by_aggregation_type(FieldAggregationMethod aggregation_type);
    static FieldType get_field_type_by_string(const std::string& str);
    static FieldAggregationMethod get_aggregation_type_by_string(const std::string& str);
    static uint32_t get_field_length_by_type(TPrimitiveType::type type, uint32_t string_length);
    bool is_row_store_column() const;
    std::string get_aggregation_name() const { return _aggregation_name; }
    bool get_result_is_nullable() const { return _result_is_nullable; }
    int get_be_exec_version() const { return _be_exec_version; }
    bool has_path_info() const { return _column_path != nullptr && !_column_path->empty(); }
    const PathInDataPtr& path_info_ptr() const { return _column_path; }
    // If it is an extracted column from variant column
    bool is_extracted_column() const {
        return _column_path != nullptr && !_column_path->empty() && _parent_col_unique_id >= 0;
    };
    std::string suffix_path() const {
        return is_extracted_column() ? _column_path->get_path() : "";
    }
    bool is_nested_subcolumn() const {
        return _column_path != nullptr && _column_path->has_nested_part();
    }
    int32_t parent_unique_id() const { return _parent_col_unique_id; }
    void set_parent_unique_id(int32_t col_unique_id) { _parent_col_unique_id = col_unique_id; }
    void set_is_bf_column(bool is_bf_column) { _is_bf_column = is_bf_column; }
    std::shared_ptr<const IDataType> get_vec_type() const;

    Status check_valid() const {
        if (type() != FieldType::OLAP_FIELD_TYPE_ARRAY &&
            type() != FieldType::OLAP_FIELD_TYPE_STRUCT &&
            type() != FieldType::OLAP_FIELD_TYPE_MAP) {
            return Status::OK();
        }
        if (is_bf_column()) {
            return Status::NotSupported("Do not support bloom filter index, type={}",
                                        get_string_by_field_type(type()));
        }
        return Status::OK();
    }

    void set_precision(int precision) {
        _precision = precision;
        _is_decimal = true;
    }

    void set_frac(int frac) { _frac = frac; }

    const VariantParams& variant_params() const { return _variant; }
    VariantParams* mutable_variant_params() { return &_variant; }

    int32_t variant_max_subcolumns_count() const { return _variant.max_subcolumns_count; }

    void set_variant_max_subcolumns_count(int32_t variant_max_subcolumns_count) {
        _variant.max_subcolumns_count = variant_max_subcolumns_count;
    }

    bool variant_is_v2() const { return _variant_is_v2; }
    void set_variant_is_v2(bool is_v2) { _variant_is_v2 = is_v2; }

    PatternTypePB pattern_type() const { return _pattern_type; }

    bool variant_enable_typed_paths_to_sparse() const {
        return _variant.enable_typed_paths_to_sparse;
    }

    int32_t variant_max_sparse_column_statistics_size() const {
        return _variant.max_sparse_column_statistics_size;
    }

    int32_t variant_sparse_hash_shard_count() const { return _variant.sparse_hash_shard_count; }

    bool variant_enable_doc_mode() const { return _variant.enable_doc_mode; }

    int64_t variant_doc_materialization_min_rows() const {
        return _variant.doc_materialization_min_rows;
    }

    int32_t variant_doc_hash_shard_count() const { return _variant.doc_hash_shard_count; }

    void set_variant_doc_materialization_min_rows(int64_t variant_doc_materialization_min_rows) {
        _variant.doc_materialization_min_rows = variant_doc_materialization_min_rows;
    }

    void set_variant_doc_hash_shard_count(int32_t variant_doc_hash_shard_count) {
        _variant.doc_hash_shard_count = variant_doc_hash_shard_count;
    }

    void set_variant_max_sparse_column_statistics_size(
            int32_t variant_max_sparse_column_statistics_size) {
        _variant.max_sparse_column_statistics_size = variant_max_sparse_column_statistics_size;
    }

    void set_variant_sparse_hash_shard_count(int32_t variant_sparse_hash_shard_count) {
        _variant.sparse_hash_shard_count = variant_sparse_hash_shard_count;
    }

    void set_variant_enable_doc_mode(bool variant_enable_doc_mode) {
        _variant.enable_doc_mode = variant_enable_doc_mode;
    }

    void set_variant_enable_typed_paths_to_sparse(bool variant_enable_typed_paths_to_sparse) {
        _variant.enable_typed_paths_to_sparse = variant_enable_typed_paths_to_sparse;
    }

    bool variant_enable_nested_group() const { return _variant.enable_nested_group; }

    void set_variant_enable_nested_group(bool val) { _variant.enable_nested_group = val; }

    bool is_decimal() const { return _is_decimal; }

private:
    int32_t _unique_id = -1;
    std::string _col_name;
    std::string _col_name_lower_case;
    // the field _type will change from TPrimitiveType
    // to string by 'EnumToString(TPrimitiveType, tcolumn.column_type.type, data_type);' (reference: TabletMeta::init_column_from_tcolumn)
    // to FieldType by 'TabletColumn::get_field_type_by_string' (reference: TabletColumn::init_from_pb).
    // And the _type in columnPB is string and it changed from FieldType by 'get_string_by_field_type' (reference: TabletColumn::to_schema_pb).
    FieldType _type;
    bool _is_key = false;
    FieldAggregationMethod _aggregation;
    std::string _aggregation_name;
    bool _is_nullable = false;
    bool _is_auto_increment = false;
    bool _is_on_update_current_timestamp {false};

    bool _has_default_value = false;
    std::string _default_value;
    bool _has_default_value_expr = false;
    std::string _default_value_expr;

    bool _is_decimal = false;
    int32_t _precision = -1;
    int32_t _frac = -1;

    int32_t _length = -1;
    int32_t _index_length = -1;

    bool _is_bf_column = false;

    bool _visible = true;

    std::vector<TabletColumnPtr> _sub_columns;
    uint32_t _sub_column_count = 0;

    bool _result_is_nullable = false;
    int _be_exec_version = -1;

    // The extracted sub-columns from "variant" contain the following information:
    int32_t _parent_col_unique_id = -1; // "variant" -> col_unique_id
    PathInDataPtr _column_path;         // the path of the sub-columns themselves
    PatternTypePB _pattern_type = PatternTypePB::MATCH_NAME_GLOB;

    VariantParams _variant;
    // TODO: Remove this transient read-schema marker after legacy ColumnVariant destinations are
    // deleted and Variant readers always produce ColumnVariantV2. It only selects the in-memory
    // compute destination and must never be serialized into tablet or segment metadata.
    bool _variant_is_v2 = false;
};

bool operator==(const TabletColumn& a, const TabletColumn& b);
bool operator!=(const TabletColumn& a, const TabletColumn& b);

class TabletIndex : public MetadataAdder<TabletIndex> {
public:
    TabletIndex() = default;
    void init_from_thrift(const TOlapTableIndex& index, const TabletSchema& tablet_schema);
    void init_from_thrift(const TOlapTableIndex& index, const std::vector<int32_t>& column_uids);
    void init_from_pb(const TabletIndexPB& index);
    void to_schema_pb(TabletIndexPB* index) const;

    int64_t index_id() const { return _index_id; }
    const std::string& index_name() const { return _index_name; }
    MOCK_FUNCTION IndexType index_type() const { return _index_type; }
    const std::vector<int32_t>& col_unique_ids() const { return _col_unique_ids; }
    MOCK_FUNCTION const std::map<std::string, std::string>& properties() const {
        return _properties;
    }
    int32_t get_gram_size() const {
        if (_properties.contains("gram_size")) {
            return std::stoi(_properties.at("gram_size"));
        }

        return 0;
    }
    int32_t get_gram_bf_size() const {
        if (_properties.contains("bf_size")) {
            return std::stoi(_properties.at("bf_size"));
        }

        return 0;
    }

    const std::string& get_index_suffix() const { return _escaped_index_suffix_path; }

    void set_escaped_escaped_index_suffix_path(const std::string& name);

    bool is_inverted_index() const { return _index_type == IndexType::INVERTED; }

    bool is_ann_index() const { return _index_type == IndexType::ANN; }

    void remove_parser_and_analyzer() {
        _properties.erase(INVERTED_INDEX_PARSER_KEY);
        _properties.erase(INVERTED_INDEX_PARSER_KEY_ALIAS);
        _properties.erase(INVERTED_INDEX_ANALYZER_NAME_KEY);
        _properties.erase(INVERTED_INDEX_NORMALIZER_NAME_KEY);
    }

    std::string field_pattern() const {
        if (_properties.contains("field_pattern")) {
            return _properties.at("field_pattern");
        }
        return "";
    }

    bool is_same_except_id(const TabletIndex* other) const {
        return _escaped_index_suffix_path == other->_escaped_index_suffix_path &&
               _index_name == other->_index_name && _index_type == other->_index_type &&
               _col_unique_ids == other->_col_unique_ids && _properties == other->_properties;
    }

private:
    int64_t _index_id = -1;
    // Identify the different index with the same _index_id
    std::string _escaped_index_suffix_path;
    std::string _index_name;
    IndexType _index_type;
    std::vector<int32_t> _col_unique_ids;
    std::map<std::string, std::string> _properties;
};

using TabletIndexPtr = std::shared_ptr<TabletIndex>;
using TabletIndexes = std::vector<std::shared_ptr<TabletIndex>>;
using PathSet = phmap::flat_hash_set<std::string>;

class TabletSchema : public MetadataAdder<TabletSchema> {
public:
    enum class ColumnType { NORMAL = 0, VARIANT = 1 };
    // TODO(yingchun): better to make constructor as private to avoid
    // manually init members incorrectly, and define a new function like
    // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema).
    TabletSchema();
    ~TabletSchema() override;

    // Init from pb
    // ignore_extracted_columns: ignore the extracted columns from variant column
    // reuse_cached_column: reuse the cached column in the schema if they are the same, to reduce memory usage
    void init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns = false,
                      bool reuse_cached_column = false);
    // Notice: Use deterministic way to serialize protobuf,
    // since serialize Map in protobuf may could lead to un-deterministic by default
    template <class PbType>
    static std::string deterministic_string_serialize(const PbType& pb) {
        std::string output;
        google::protobuf::io::StringOutputStream string_output_stream(&output);
        google::protobuf::io::CodedOutputStream output_stream(&string_output_stream);
        output_stream.SetSerializationDeterministic(true);
        pb.SerializeToCodedStream(&output_stream);
        return output;
    }
    void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const;
    void append_column(TabletColumn column, ColumnType col_type = ColumnType::NORMAL);
    void append_index(TabletIndex&& index);
    void remove_index(int64_t index_id);
    void clear_index();
    // Must make sure the row column is always the last column
    void add_row_column();
    void copy_from(const TabletSchema& tablet_schema);
    // lightweight copy, take care of lifecycle of TabletColumn
    void shawdow_copy_without_columns(const TabletSchema& tablet_schema);
    void update_index_info_from(const TabletSchema& tablet_schema);
    std::string to_key() const;
    // get_metadata_size is only the memory of the TabletSchema itself, not include child objects.
    int64_t mem_size() const { return get_metadata_size(); }
    size_t row_size() const;
    int32_t field_index(const std::string& field_name) const;
    int32_t field_index(const PathInData& path) const;
    int32_t field_index(int32_t col_unique_id) const;
    const TabletColumn& column(size_t ordinal) const;
    Result<const TabletColumn*> column(const std::string& field_name) const;
    Status have_column(const std::string& field_name) const;
    bool exist_column(const std::string& field_name) const;
    bool has_column_unique_id(int32_t col_unique_id) const;
    const TabletColumn& column_by_uid(int32_t col_unique_id) const;
    TabletColumn& mutable_column_by_uid(int32_t col_unique_id);
    TabletColumn& mutable_column(size_t ordinal);
    void replace_column(size_t pos, TabletColumn new_col);
    const std::vector<TabletColumnPtr>& columns() const;
    size_t num_columns() const { return _num_columns; }
    size_t num_visible_columns() const {
        return std::count_if(_cols.begin(), _cols.end(),
                             [](const TabletColumnPtr& column) { return column->visible(); });
    }
    size_t num_visible_value_columns() const {
        return std::count_if(_cols.begin(), _cols.end(), [](const TabletColumnPtr& column) {
            return column->visible() && !column->is_key();
        });
    }
    // num_key_columns: Total number of sort key columns in the table, determined by the key columns
    // specified in DUPLICATE KEY/UNIQUE KEY/AGGREGATE KEY when creating the table, used for complete data sorting
    // Example: CREATE TABLE t(a INT, b DATE, c VARCHAR) DUPLICATE KEY(a, b, c)
    //          Then num_key_columns = 3 (columns a, b, c are all sort keys)
    size_t num_key_columns() const { return _num_key_columns; }
    const std::vector<uint32_t>& cluster_key_uids() const { return _cluster_key_uids; }
    size_t num_null_columns() const { return _num_null_columns; }
    // num_short_key_columns: Number of columns used to build the Short Key Index, automatically calculated by FE
    // Limited by max column count (default 3) and max bytes (default 36 bytes). Types like float/double/STRING/JSONB
    // cannot be used as short keys. VARCHAR can only be the last short key column. Optimizes index size and query performance.
    // Example: CREATE TABLE t(a INT, b DATE, c VARCHAR) DUPLICATE KEY(a, b, c)
    //          Then num_short_key_columns = 3 (a, b, c all meet criteria, c as VARCHAR is the last short key)
    // Example: CREATE TABLE t(a INT, b DOUBLE, c DATE) DUPLICATE KEY(a, b, c)
    //          Then num_short_key_columns = 1 (b is DOUBLE type which cannot be short key, stops at b)
    // short key's size is limited to 36 bytes, because it will be loaded to memory during segment loaded.
    size_t num_short_key_columns() const { return _num_short_key_columns; }
    size_t num_rows_per_row_block() const { return _num_rows_per_row_block; }
    size_t num_variant_columns() const { return _num_variant_columns; };
    size_t num_virtual_columns() const { return _num_virtual_columns; }
    KeysType keys_type() const { return _keys_type; }
    SortType sort_type() const { return _sort_type; }
    size_t sort_col_num() const { return _sort_col_num; }
    CompressKind compress_kind() const { return _compress_kind; }
    size_t next_column_unique_id() const { return _next_column_unique_id; }
    bool has_bf_fpp() const { return _has_bf_fpp; }
    double bloom_filter_fpp() const { return _bf_fpp; }
    bool is_in_memory() const { return _is_in_memory; }
    void set_is_in_memory(bool is_in_memory) { _is_in_memory = is_in_memory; }
    void set_disable_auto_compaction(bool disable_auto_compaction) {
        _disable_auto_compaction = disable_auto_compaction;
    }
    bool disable_auto_compaction() const { return _disable_auto_compaction; }
    // Deprecated legacy switch for flatten-nested variant behavior.
    // It is distinct from variant_enable_nested_group.
    void set_deprecated_variant_flatten_nested(bool flatten_nested) {
        _deprecated_enable_variant_flatten_nested = flatten_nested;
    }
    bool deprecated_variant_flatten_nested() const {
        return _deprecated_enable_variant_flatten_nested;
    }
    // indicate if full row store column(all the columns encodes as row) exists
    bool has_row_store_for_all_columns() const {
        return _store_row_column && row_columns_uids().empty();
    }
    void set_skip_write_index_on_load(bool skip) { _skip_write_index_on_load = skip; }
    bool skip_write_index_on_load() const { return _skip_write_index_on_load; }
    int32_t delete_sign_idx() const { return _delete_sign_idx; }
    bool has_sequence_col() const { return _sequence_col_idx != -1; }
    int32_t sequence_col_idx() const { return _sequence_col_idx; }
    void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; }
    int32_t version_col_idx() const { return _version_col_idx; }
    bool has_skip_bitmap_col() const { return _skip_bitmap_col_idx != -1; }
    int32_t skip_bitmap_col_idx() const { return _skip_bitmap_col_idx; }
    bool is_tso_enabled() const { return _commit_tso_col_idx != -1 || _binlog_tso_col_idx != -1; }
    int32_t commit_tso_col_idx() const { return _commit_tso_col_idx; }
    int32_t binlog_tso_col_idx() const { return _binlog_tso_col_idx; }
    int32_t binlog_lsn_col_idx() const { return _binlog_lsn_col_idx; }
    int32_t binlog_op_col_idx() const { return _binlog_op_col_idx; }
    segment_v2::CompressionTypePB compression_type() const { return _compression_type; }
    void set_row_store_page_size(long page_size) { _row_store_page_size = page_size; }
    long row_store_page_size() const { return _row_store_page_size; }
    void set_storage_page_size(long storage_page_size) { _storage_page_size = storage_page_size; }
    long storage_page_size() const { return _storage_page_size; }
    void set_storage_dict_page_size(long storage_dict_page_size) {
        _storage_dict_page_size = storage_dict_page_size;
    }
    long storage_dict_page_size() const { return _storage_dict_page_size; }
    bool has_global_row_id() const {
        for (auto [col_name, _] : _field_name_to_index) {
            if (col_name.start_with(StringRef(BeConsts::GLOBAL_ROWID_COL.data(),
                                              BeConsts::GLOBAL_ROWID_COL.size()))) {
                return true;
            }
        }
        return false;
    }

    const std::vector<const TabletIndex*> inverted_indexes() const {
        std::vector<const TabletIndex*> inverted_indexes;
        for (const auto& index : _indexes) {
            if (index->index_type() == IndexType::INVERTED) {
                inverted_indexes.emplace_back(index.get());
            }
        }
        return inverted_indexes;
    }
    // True when anything at all lives in this schema's inverted index FILE.
    //
    // Spelled out as `has_inverted_index() || has_ann_index()` at ~27 call sites
    // before this existed -- rowset writers, segment creation, segcompaction,
    // snapshot and migration, and most of the cloud paths. Every one of them is
    // asking the same question ("is there an index file to carry, warm, link or
    // rewrite?"), and each open-coded copy is a place a third index type would
    // have to be remembered.
    bool has_inverted_or_ann_index() const { return has_inverted_index() || has_ann_index(); }

    // Both index types that live in the inverted index FILE, in schema order.
    // Every storage format stores them together: V1/V2/V3 as CLucene directories,
    // SNII as text metadata groups plus an ANN blob logical index. A rewrite that
    // enumerated only the inverted ones would seal a file missing every ANN index
    // while the schema still claimed them.
    const std::vector<const TabletIndex*> inverted_and_ann_indexes() const {
        std::vector<const TabletIndex*> indexes;
        for (const auto& index : _indexes) {
            if (index->index_type() == IndexType::INVERTED ||
                index->index_type() == IndexType::ANN) {
                indexes.emplace_back(index.get());
            }
        }
        return indexes;
    }
    bool has_inverted_index() const {
        for (const auto& index : _indexes) {
            DBUG_EXECUTE_IF("tablet_schema::has_inverted_index", {
                if (index->col_unique_ids().empty()) {
                    throw Exception(Status::InternalError("col unique ids cannot be empty"));
                }
            });

            if (index->index_type() == IndexType::INVERTED) {
                //if index_id == -1, ignore it.
                if (!index->col_unique_ids().empty() && index->col_unique_ids()[0] >= 0) {
                    return true;
                }
            }
        }
        return false;
    }

    bool has_ann_index() const {
        for (const auto& index : _indexes) {
            if (index->index_type() == IndexType::ANN) {
                if (!index->col_unique_ids().empty() && index->col_unique_ids()[0] >= 0) {
                    return true;
                }
            }
        }
        return false;
    }

    bool has_inverted_index_with_index_id(int64_t index_id) const;

    std::vector<const TabletIndex*> inverted_indexs(const TabletColumn& col) const;

    std::vector<const TabletIndex*> inverted_indexs(int32_t col_unique_id,
                                                    const std::string& suffix_path = "") const;
    const TabletIndex* ann_index(const TabletColumn& col) const;

    // Regardless of whether this column supports inverted index
    // TabletIndex information will be returned as long as it exists.
    const TabletIndex* ann_index(int32_t col_unique_id, const std::string& suffix_path = "") const;

    std::vector<TabletIndexPtr> inverted_index_by_field_pattern(
            int32_t col_unique_id, const std::string& field_pattern) const;

    bool has_ngram_bf_index(int32_t col_unique_id) const;
    const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const;
    double get_bloom_filter_fpp(int32_t col_unique_id) const;
    double get_bloom_filter_fpp(const TabletColumn& column) const;
    const TabletIndex* get_index(int32_t col_unique_id, IndexType index_type,
                                 const std::string& suffix_path) const;
    void update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>& indexes);
    // If schema version is not set, it should be -1
    int32_t schema_version() const { return _schema_version; }
    void clear_columns();
    // Each column id is an ordinal in TabletSchema::_cols, not a unique id or ReadSchema ordinal.
    // The resulting Block uses the selected physical TabletSchema column types.
    Block create_storage_block(const std::vector<uint32_t>& column_ids) const;
    Block create_storage_block() const;
    void set_schema_version(int32_t version) { _schema_version = version; }
    void set_auto_increment_column(const std::string& auto_increment_column) {
        _auto_increment_column = auto_increment_column;
    }
    std::string auto_increment_column() const { return _auto_increment_column; }

    void set_table_id(int64_t table_id) { _table_id = table_id; }
    int64_t table_id() const { return _table_id; }
    void set_db_id(int64_t db_id) { _db_id = db_id; }
    int64_t db_id() const { return _db_id; }
    void build_current_tablet_schema(int64_t index_id, int32_t version,
                                     const OlapTableIndexSchema* index,
                                     const TabletSchema& out_tablet_schema);

    // copy extracted columns from src_schema
    void copy_extracted_columns(const TabletSchema& src_schema);

    // only reserve extracted columns
    void reserve_extracted_columns();

    std::string get_all_field_names() const {
        std::string str = "[";
        for (auto p : _field_name_to_index) {
            if (str.size() > 1) {
                str += ", ";
            }
            str += p.first.to_string() + "(" + std::to_string(_cols[p.second]->unique_id()) + ")";
        }
        str += "]";
        return str;
    }

    // Dump [(name, type, is_nullable), ...]
    std::string dump_structure() const {
        std::string str = "[";
        for (auto p : _cols) {
            if (str.size() > 1) {
                str += ", ";
            }
            str += "(";
            str += p->name();
            str += ", ";
            str += TabletColumn::get_string_by_field_type(p->type());
            str += ", ";
            str += "is_nullable:";
            str += (p->is_nullable() ? "true" : "false");
            str += ")";
        }
        str += "]";
        return str;
    }

    std::string dump_full_schema() const {
        std::string str = "[";
        for (auto p : _cols) {
            if (str.size() > 1) {
                str += ", ";
            }
            ColumnPB col_pb;
            p->to_schema_pb(&col_pb);
            str += "(";
            str += col_pb.ShortDebugString();
            str += ")";
        }
        str += "]";
        return str;
    }

    std::shared_ptr<TabletSchema> copy_without_variant_extracted_columns();
    InvertedIndexStorageFormatPB get_inverted_index_storage_format() const {
        return _inverted_index_storage_format;
    }

    void update_tablet_columns(const TabletSchema& tablet_schema,
                               const std::vector<TColumn>& t_columns);

    const std::vector<int32_t>& row_columns_uids() const { return _row_store_column_unique_ids; }

    int64_t get_metadata_size() const override;

    struct SubColumnInfo {
        TabletColumn column;
        TabletIndexes indexes;
    };

    // all path in path_set_info are relative to the parent column
    struct PathsSetInfo {
        std::unordered_map<std::string, SubColumnInfo> typed_path_set;    // typed columns
        std::unordered_map<std::string, TabletIndexes> subcolumn_indexes; // subcolumns indexes
        PathSet sub_path_set;                                             // extracted columns
        PathSet sparse_path_set;                                          // sparse columns

        // "Materialized regular path" means compaction chose to store this path as a dedicated
        // column in the schema, either typed or extracted, instead of re-emitting it dynamically.
        bool contains_materialized_regular_path(const std::string& path) const {
            return typed_path_set.contains(path) || sub_path_set.contains(path);
        }
    };

    void set_path_set_info(std::unordered_map<int32_t, PathsSetInfo>&& path_set_info_map) {
        _path_set_info_map = std::move(path_set_info_map);
    }

    const PathsSetInfo& path_set_info(int32_t unique_id) const {
        return _path_set_info_map.at(unique_id);
    }

    const PathsSetInfo* try_path_set_info(int32_t unique_id) const {
        auto it = _path_set_info_map.find(unique_id);
        return it == _path_set_info_map.end() ? nullptr : &it->second;
    }

    bool need_record_variant_extended_schema() const { return variant_max_subcolumns_count() == 0; }

    int32_t variant_max_subcolumns_count() const {
        for (const auto& col : _cols) {
            if (col->is_variant_type()) {
                return col->variant_max_subcolumns_count();
            }
        }
        return 0;
    }
    const std::unordered_map<uint32_t, std::vector<uint32_t>>& seq_col_idx_to_value_cols_idx()
            const {
        return _seq_col_idx_to_value_cols_idx;
    }

    bool has_seq_map() const { return !_seq_col_idx_to_value_cols_idx.empty(); }

    TabletStorageFormatPB storage_format() const { return _storage_format; }
    void set_storage_format(TabletStorageFormatPB v) { _storage_format = v; }

private:
    friend bool operator==(const TabletSchema& a, const TabletSchema& b);
    friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
    TabletSchema(const TabletSchema&) = default;

    KeysType _keys_type = DUP_KEYS;
    SortType _sort_type = SortType::LEXICAL;
    size_t _sort_col_num = 0;
    std::vector<TabletColumnPtr> _cols;

    std::vector<TabletIndexPtr> _indexes;
    std::unordered_map<StringRef, int32_t, StringRefHash> _field_name_to_index;
    std::unordered_map<int32_t, int32_t> _field_uniqueid_to_index;
    std::unordered_map<PathInDataRef, int32_t, PathInDataRef::Hash> _field_path_to_index;

    // index_type/col_unique_id/suffix -> idxs in _indexes
    using IndexKey = std::tuple<IndexType, int32_t, std::string>;
    struct IndexKeyHash {
        size_t operator()(const IndexKey& t) const {
            uint32_t seed = 0;
            seed = doris::HashUtil::hash((const char*)&std::get<0>(t), sizeof(std::get<0>(t)),
                                         seed);
            seed = doris::HashUtil::hash((const char*)&std::get<1>(t), sizeof(std::get<1>(t)),
                                         seed);
            seed = doris::HashUtil::hash((const char*)std::get<2>(t).c_str(),
                                         static_cast<uint32_t>(std::get<2>(t).size()), seed);
            return seed;
        }
    };
    std::unordered_map<IndexKey, std::vector<size_t>, IndexKeyHash> _col_id_suffix_to_index;

    int32_t _num_columns = 0;
    size_t _num_variant_columns = 0;
    size_t _num_virtual_columns = 0;
    size_t _num_key_columns = 0;
    std::vector<uint32_t> _cluster_key_uids;
    size_t _num_null_columns = 0;
    size_t _num_short_key_columns = 0;
    size_t _num_rows_per_row_block = 0;
    CompressKind _compress_kind = COMPRESS_NONE;
    segment_v2::CompressionTypePB _compression_type = segment_v2::CompressionTypePB::LZ4F;
    long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
    long _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
    long _storage_dict_page_size = segment_v2::STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
    size_t _next_column_unique_id = 0;
    std::string _auto_increment_column;

    bool _has_bf_fpp = false;
    double _bf_fpp = 0;
    bool _is_in_memory = false;
    int32_t _delete_sign_idx = -1;
    int32_t _sequence_col_idx = -1;
    int32_t _version_col_idx = -1;
    int32_t _skip_bitmap_col_idx = -1;
    int32_t _commit_tso_col_idx = -1;
    int32_t _binlog_tso_col_idx = -1;
    int32_t _binlog_lsn_col_idx = -1;
    int32_t _binlog_op_col_idx = -1;
    int32_t _schema_version = -1;
    int64_t _table_id = -1;
    int64_t _db_id = -1;
    bool _disable_auto_compaction = false;
    bool _store_row_column = false;
    bool _skip_write_index_on_load = false;
    InvertedIndexStorageFormatPB _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1;

    // Contains column ids of which columns should be encoded into row store.
    // ATTN: For compability reason empty cids means all columns of tablet schema are encoded to row column
    std::vector<int32_t> _row_store_column_unique_ids;
    bool _deprecated_enable_variant_flatten_nested = false;

    std::map<size_t, int32_t> _vir_col_idx_to_unique_id;

    // value: extracted path set and sparse path set
    std::unordered_map<int32_t, PathsSetInfo> _path_set_info_map;

    // key: field_pattern
    // value: indexes
    using PatternToIndex = std::unordered_map<std::string, std::vector<TabletIndexPtr>>;
    std::unordered_map<int32_t, PatternToIndex> _index_by_unique_id_with_pattern;

    // Default behavior for new segments: use external ColumnMeta region + CMO table if true
    // Persisted tablet storage format. Authoritative source for "is this tablet V3?"
    // decisions in the segment write paths. Old PBs without this field are upgraded in
    // init_from_pb() by deriving V3 from any of the three legacy V3-flavor flags.
    TabletStorageFormatPB _storage_format {TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2};
    // Sequence column unique id mapping to value columns unique id
    std::unordered_map<uint32_t, std::vector<uint32_t>> _seq_col_uid_to_value_cols_uid;
    // Value column unique id mapping to sequence column unique id(also map sequence column it self)
    std::unordered_map<uint32_t, uint32_t> _value_col_uid_to_seq_col_uid;
    // Sequence column index mapping to value column index
    std::unordered_map<uint32_t, std::vector<uint32_t>> _seq_col_idx_to_value_cols_idx;
};

bool operator==(const TabletSchema& a, const TabletSchema& b);
bool operator!=(const TabletSchema& a, const TabletSchema& b);

using TabletSchemaSPtr = std::shared_ptr<TabletSchema>;

} // namespace doris

Coverage Report

Created: 2026-08-26 03:13