be/src/storage/segment/encoding_info.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "storage/segment/encoding_info.h"

#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>

#include <array>
#include <iterator>
#include <type_traits>
#include <unordered_map>
#include <utility>

#include "common/config.h"
#include "runtime/exec_env.h"
#include "storage/olap_common.h"
#include "storage/segment/binary_dict_page.h"
#include "storage/segment/binary_dict_page_pre_decoder.h"
#include "storage/segment/binary_plain_page.h"
#include "storage/segment/binary_plain_page_v2.h"
#include "storage/segment/binary_plain_page_v2_pre_decoder.h"
#include "storage/segment/binary_plain_page_v3.h"
#include "storage/segment/binary_plain_page_v3_pre_decoder.h"
#include "storage/segment/binary_prefix_page.h"
#include "storage/segment/bitshuffle_page.h"
#include "storage/segment/bitshuffle_page_pre_decoder.h"
#include "storage/segment/frame_of_reference_page.h"
#include "storage/segment/options.h"
#include "storage/segment/plain_page.h"
#include "storage/segment/rle_page.h"
#include "storage/types.h"

namespace doris {
namespace segment_v2 {

template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, PLAIN_ENCODING, CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return PlainPageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new PlainPageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type>
struct TypeEncodingTraits<type, PLAIN_ENCODING, Slice> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BinaryPlainPageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BinaryPlainPageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, PLAIN_ENCODING_V2, CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return PlainPageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new PlainPageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type>
struct TypeEncodingTraits<type, PLAIN_ENCODING_V2, Slice> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BinaryPlainPageV2Builder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BinaryPlainPageV2Decoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, PLAIN_ENCODING_V3, CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return PlainPageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new PlainPageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type>
struct TypeEncodingTraits<type, PLAIN_ENCODING_V3, Slice> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BinaryPlainPageV3Builder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BinaryPlainPageV3Decoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, BIT_SHUFFLE, CppType,
                          typename std::enable_if<!std::is_same<CppType, Slice>::value>::type> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BitshufflePageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BitShufflePageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <>
struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_BOOL, RLE, uint8_t> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return RlePageBuilder<FieldType::OLAP_FIELD_TYPE_BOOL>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new RlePageDecoder<FieldType::OLAP_FIELD_TYPE_BOOL>(data, opts);
        return Status::OK();
    }
};

template <FieldType type>
struct TypeEncodingTraits<type, DICT_ENCODING, Slice> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BinaryDictPageBuilder::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BinaryDictPageDecoder(data, opts);
        return Status::OK();
    }
};

template <>
struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING,
                          typename CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATE>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new FrameOfReferencePageDecoder<FieldType::OLAP_FIELD_TYPE_DATE>(data, opts);
        return Status::OK();
    }
};

template <>
struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING,
                          typename CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATEV2>::create(builder,
                                                                                      opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new FrameOfReferencePageDecoder<FieldType::OLAP_FIELD_TYPE_DATEV2>(data, opts);
        return Status::OK();
    }
};

template <>
struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING,
                          typename CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::create(builder,
                                                                                          opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder =
                new FrameOfReferencePageDecoder<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>(data, opts);
        return Status::OK();
    }
};

template <>
struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING,
                          typename CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::CppType> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::create(builder,
                                                                                           opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder =
                new FrameOfReferencePageDecoder<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>(data, opts);
        return Status::OK();
    }
};

template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, FOR_ENCODING, CppType,
                          typename std::enable_if<IsIntegral<CppType>::value>::type> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return FrameOfReferencePageBuilder<type>::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new FrameOfReferencePageDecoder<type>(data, opts);
        return Status::OK();
    }
};

template <FieldType type>
struct TypeEncodingTraits<type, PREFIX_ENCODING, Slice> {
    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
        return BinaryPrefixPageBuilder::create(builder, opts);
    }
    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                      PageDecoder** decoder) {
        *decoder = new BinaryPrefixPageDecoder(data, opts);
        return Status::OK();
    }
};

EncodingInfoResolver::EncodingInfoResolver() {
    _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, FOR_ENCODING, true>();
    _add_map<FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, FOR_ENCODING, true>();
    _add_map<FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_INT, FOR_ENCODING, true>();
    _add_map<FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, FOR_ENCODING, true>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();

    _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_LARGEINT, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();

    _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V3>();
    _add_map<FieldType::OLAP_FIELD_TYPE_CHAR, PREFIX_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V3>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V3>();
    _add_map<FieldType::OLAP_FIELD_TYPE_STRING, PREFIX_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V3>();
    _add_map<FieldType::OLAP_FIELD_TYPE_JSONB, PREFIX_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V3>();
    _add_map<FieldType::OLAP_FIELD_TYPE_VARIANT, PREFIX_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, RLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE>();
    _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, true>();

    _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V3>();

    _add_map<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V3>();

    _add_map<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V3>();

    _add_map<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING>();
    _add_map<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2>();
    _add_map<FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V3>();
}

EncodingInfoResolver::~EncodingInfoResolver() {
    for (auto& it : _encoding_map) {
        delete it.second;
    }
    _encoding_map.clear();
}

namespace {
bool is_integer_type(FieldType type) {
    return type == FieldType::OLAP_FIELD_TYPE_TINYINT ||
           type == FieldType::OLAP_FIELD_TYPE_SMALLINT || type == FieldType::OLAP_FIELD_TYPE_INT ||
           type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT;
}

bool is_binary_type(FieldType type) {
    return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
           type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB ||
           type == FieldType::OLAP_FIELD_TYPE_VARIANT || type == FieldType::OLAP_FIELD_TYPE_HLL ||
           type == FieldType::OLAP_FIELD_TYPE_BITMAP ||
           type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
           type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
}
} // namespace

EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type,
                                                          EncodingPreference encoding_preference,
                                                          bool optimize_value_seek) const {
    // Predicate for default encoding transformation
    // Parameters: (type, current_default_encoding, optimize_value_seek)
    // Returns: true if the transformation should be applied
    using Predicate = std::function<bool(FieldType, EncodingTypePB,
                                         EncodingPreference encoding_preference, bool)>;

    // Hook for transforming default encoding: predicate -> target encoding
    struct EncodingTransform {
        Predicate predicate;
        EncodingTypePB target_encoding;
    };

    // Static array of hooks for default encoding transformations
    static const std::vector<EncodingTransform> hooks = {
            // Hook 1: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V2
            // Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v2
            EncodingTransform {
                    .predicate =
                            [](FieldType type, EncodingTypePB encoding,
                               EncodingPreference encoding_preference, bool optimize_value_seek) {
                                return encoding == PLAIN_ENCODING && is_binary_type(type) &&
                                       encoding_preference.binary_plain_encoding_default_impl ==
                                               BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2;
                            },
                    .target_encoding = PLAIN_ENCODING_V2},

            // Hook 1b: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V3
            // Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v3.
            // V3 hook is placed before the v2 hook in the future would be wrong; both keys
            // are mutually exclusive at the source enum, so order is informational only.
            EncodingTransform {
                    .predicate =
                            [](FieldType type, EncodingTypePB encoding,
                               EncodingPreference encoding_preference, bool optimize_value_seek) {
                                return encoding == PLAIN_ENCODING && is_binary_type(type) &&
                                       encoding_preference.binary_plain_encoding_default_impl ==
                                               BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V3;
                            },
                    .target_encoding = PLAIN_ENCODING_V3},

            // Hook 2: Integer types - any encoding -> PLAIN_ENCODING
            // Applies when: type is integer and config enables plain encoding for integers
            EncodingTransform {
                    .predicate =
                            [](FieldType type, EncodingTypePB encoding,
                               EncodingPreference encoding_preference, bool optimize_value_seek) {
                                return is_integer_type(type) &&
                                       encoding_preference.integer_type_default_use_plain_encoding;
                            },
                    .target_encoding = PLAIN_ENCODING}};

    auto& encoding_map =
            optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map;
    auto it = encoding_map.find(type);
    if (it != encoding_map.end()) {
        EncodingTypePB encoding = it->second;

        // Apply hooks in order to transform the default encoding
        for (const auto& hook : hooks) {
            if (hook.predicate(type, encoding, encoding_preference, optimize_value_seek)) {
                // Verify target encoding is available for this type
                if (_encoding_map.contains(std::make_pair(type, hook.target_encoding))) {
                    encoding = hook.target_encoding;
                    break; // Apply only the first matching hook
                }
            }
        }

        return encoding;
    }
    return UNKNOWN_ENCODING;
}

Status EncodingInfoResolver::get(FieldType data_type, EncodingTypePB encoding_type,
                                 EncodingPreference encoding_preference, const EncodingInfo** out) {
    if (encoding_type == DEFAULT_ENCODING) {
        encoding_type = get_default_encoding(data_type, encoding_preference, false);
    }
    auto key = std::make_pair(data_type, encoding_type);
    auto it = _encoding_map.find(key);
    if (it == std::end(_encoding_map)) {
        return Status::InternalError("fail to find valid type encoding, type:{}, encoding:{}",
                                     data_type, encoding_type);
    }
    *out = it->second;
    return Status::OK();
}

template <typename TraitsClass>
EncodingInfo::EncodingInfo(TraitsClass traits)
        : _create_builder_func(TraitsClass::create_page_builder),
          _create_decoder_func(TraitsClass::create_page_decoder),
          _type(TraitsClass::type),
          _encoding(TraitsClass::encoding) {
    if (_encoding == BIT_SHUFFLE) {
        _data_page_pre_decoder = std::make_unique<BitShufflePagePreDecoder>();
    } else if (_encoding == DICT_ENCODING) {
        _data_page_pre_decoder = std::make_unique<BinaryDictPagePreDecoder>();
    } else if (_encoding == PLAIN_ENCODING_V2) {
        // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2
        // to convert varint-encoded lengths to offset array format
        if constexpr (std::is_same_v<typename TraitsClass::CppType, Slice>) {
            _data_page_pre_decoder = std::make_unique<BinaryPlainPageV2PreDecoder>();
        }
    } else if (_encoding == PLAIN_ENCODING_V3) {
        if constexpr (std::is_same_v<typename TraitsClass::CppType, Slice>) {
            _data_page_pre_decoder = std::make_unique<BinaryPlainPageV3PreDecoder>();
        }
    }
}

#ifdef BE_TEST
static EncodingInfoResolver s_encoding_info_resolver;
#endif

Status EncodingInfo::get(FieldType type, EncodingTypePB encoding_type,
                         EncodingPreference encoding_preference, const EncodingInfo** out) {
#ifdef BE_TEST
    return s_encoding_info_resolver.get(type, encoding_type, encoding_preference, out);
#else
    auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
    if (resolver == nullptr) {
        return Status::InternalError("EncodingInfoResolver not initialized");
    }
    return resolver->get(type, encoding_type, encoding_preference, out);
#endif
}

EncodingTypePB EncodingInfo::get_default_encoding(FieldType type,
                                                  EncodingPreference encoding_preference,
                                                  bool optimize_value_seek) {
#ifdef BE_TEST
    return s_encoding_info_resolver.get_default_encoding(type, encoding_preference,
                                                         optimize_value_seek);
#else
    auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver();
    if (resolver == nullptr) {
        return UNKNOWN_ENCODING;
    }
    return resolver->get_default_encoding(type, encoding_preference, optimize_value_seek);
#endif
}

Status EncodingInfo::create_page_builder(const PageBuilderOptions& opts,
                                         std::unique_ptr<PageBuilder>& builder) const {
    PageBuilder* raw_builder = nullptr;
    RETURN_IF_ERROR(_create_builder_func(opts, &raw_builder));
    builder.reset(raw_builder);
    return Status::OK();
}

Status EncodingInfo::create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
                                         std::unique_ptr<PageDecoder>& decoder) const {
    PageDecoder* raw_decoder = nullptr;
    RETURN_IF_ERROR(_create_decoder_func(data, opts, &raw_decoder));
    decoder.reset(raw_decoder);
    return Status::OK();
}

} // namespace segment_v2
} // namespace doris

Coverage Report

Created: 2026-05-25 13:19