be/src/storage/segment/column_reader.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "storage/segment/column_reader.h"

#include <assert.h>
#include <gen_cpp/Descriptors_constants.h>
#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <glog/logging.h>

#include <algorithm>
#include <memory>
#include <ostream>
#include <set>
#include <string>
#include <utility>

#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/binary_cast.hpp"
#include "core/column/column.h"
#include "core/column/column_array.h"
#include "core/column/column_map.h"
#include "core/column/column_nullable.h"
#include "core/column/column_struct.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_agg_state.h"
#include "core/data_type/data_type_factory.hpp"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/define_primitive_type.h"
#include "core/decimal12.h"
#include "core/string_ref.h"
#include "core/types.h"
#include "core/value/decimalv2_value.h"
#include "core/value/vdatetime_value.h" //for VecDateTime
#include "io/fs/file_reader.h"
#include "storage/index/ann/ann_index_reader.h"
#include "storage/index/bloom_filter/bloom_filter.h"
#include "storage/index/bloom_filter/bloom_filter_index_reader.h"
#include "storage/index/index_file_reader.h"
#include "storage/index/index_reader.h"
#include "storage/index/inverted/analyzer/analyzer.h"
#include "storage/index/inverted/inverted_index_reader.h"
#include "storage/index/zone_map/zone_map_index.h"
#include "storage/iterators.h"
#include "storage/olap_common.h"
#include "storage/predicate/block_column_predicate.h"
#include "storage/predicate/column_predicate.h"
#include "storage/segment/binary_dict_page.h" // for BinaryDictPageDecoder
#include "storage/segment/binary_plain_page.h"
#include "storage/segment/column_meta_accessor.h"
#include "storage/segment/encoding_info.h" // for EncodingInfo
#include "storage/segment/page_decoder.h"
#include "storage/segment/page_handle.h" // for PageHandle
#include "storage/segment/page_io.h"
#include "storage/segment/page_pointer.h" // for PagePointer
#include "storage/segment/row_ranges.h"
#include "storage/segment/segment.h"
#include "storage/segment/segment_prefetcher.h"
#include "storage/segment/variant/variant_column_reader.h"
#include "storage/tablet/tablet_schema.h"
#include "storage/types.h" // for TypeInfo
#include "util/bitmap.h"
#include "util/block_compression.h"
#include "util/concurrency_stats.h"
#include "util/defer_op.h"
#include "util/rle_encoding.h" // for RleDecoder
#include "util/slice.h"

namespace doris::segment_v2 {
#include "storage/segment/column_reader.h"

inline bool read_as_string(PrimitiveType type) {
    return type == PrimitiveType::TYPE_STRING || type == PrimitiveType::INVALID_TYPE ||
           type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT;
}

bool is_meta_access_path_component(const std::string& component) {
    return StringCaseEqual()(component, ColumnIterator::ACCESS_OFFSET) ||
           StringCaseEqual()(component, ColumnIterator::ACCESS_NULL);
}

bool uses_legacy_access_path_encoding(const TColumnAccessPath& path) {
    return !path.__isset.version ||
           path.version == g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY;
}

namespace {

// Nested access paths are processed one container level at a time:
//
// 1. Each Map/Array/Struct iterator's set_access_paths() calls _prepare_nested_access_paths(). For
//    the all-path and predicate-path channels independently, _split_access_paths() validates the
//    wire encoding and type-selected payload, removes the current iterator name, and separates
//    requests consumed by the current iterator from paths that still address a data descendant. A
//    current DATA request requires all data children. Struct owns current-level NULL metadata;
//    Map and Array own NULL and OFFSET metadata. A supported metadata-only request can stop before
//    descendant routing and mark every data child SKIP.
// 2. This router interprets only the first remaining component and routes the path according to
//    the container topology:
//    - Struct components already name fields. Select the paths for each field without rewriting.
//    - Array `*` names its only item. Retarget `*` to the item iterator name.
//    - Map `KEYS` and `VALUES` directly name its logical children. Map `*` creates a complete DATA
//      path for `KEYS` and routes any trailing components through `VALUES`.
// 3. The container forwards the routed all-path and predicate-path channels to each selected
//    child's set_access_paths(), then finalizes that child's PREDICATE/LAZY_OUTPUT/SKIP
//    requirement. The child repeats the same flow, which handles arbitrary Map/Array/Struct
//    nesting.
//
// At this point versions have been validated, legacy paths have DATA type, and every payload
// selected by type is non-empty. Routing never interprets or rewrites an unselected compatibility
// payload.
class DescendantAccessPathRouter final {
public:
    DescendantAccessPathRouter() = delete;

    // Preserve the two set_access_paths() input channels. all_paths originates from the
    // all-access-path superset, while predicate_paths separately records predicate-phase paths.
    // Routing may omit all_paths when the parent already requires complete child data.
    struct ChildAccessPaths {
        TColumnAccessPaths all_paths;
        TColumnAccessPaths predicate_paths;

        bool empty() const { return all_paths.empty() && predicate_paths.empty(); }
    };

    struct MapChildAccessPaths {
        ChildAccessPaths key;
        ChildAccessPaths value;
    };

    // Map children use the logical KEYS/VALUES selectors as their access-path names, independent
    // of physical child column names. Expand a wildcard to complete keys and route its trailing
    // qualifiers to values.
    static Result<MapChildAccessPaths> route_map_paths_to_children(
            TColumnAccessPaths all_paths, TColumnAccessPaths predicate_paths) {
        MapChildAccessPaths child_paths;
        auto status = distribute_map_paths(std::move(all_paths), child_paths.key.all_paths,
                                           child_paths.value.all_paths);
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        status = distribute_map_paths(std::move(predicate_paths), child_paths.key.predicate_paths,
                                      child_paths.value.predicate_paths);
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        return child_paths;
    }

    // Array has one data child. Retarget its logical wildcard selector to the item iterator name.
    static Result<ChildAccessPaths> route_array_paths_to_item(TColumnAccessPaths all_paths,
                                                              TColumnAccessPaths predicate_paths,
                                                              const std::string& item_name) {
        ChildAccessPaths child_paths {.all_paths = std::move(all_paths),
                                      .predicate_paths = std::move(predicate_paths)};
        auto retarget_wildcard_paths_to_item = [&](TColumnAccessPaths& paths) -> Status {
            for (auto& path : paths) {
                const bool is_wildcard = DORIS_TRY(selected_payload_head_matches(
                        path, ColumnIterator::ACCESS_ALL, PathHeadMatchMode::EXACT));
                if (is_wildcard) {
                    RETURN_IF_ERROR(replace_selected_payload_head(path, item_name));
                }
            }
            return Status::OK();
        };

        auto status = retarget_wildcard_paths_to_item(child_paths.all_paths);
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        status = retarget_wildcard_paths_to_item(child_paths.predicate_paths);
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        return child_paths;
    }

    // Struct selectors already use child field names. Select the paths for one child without
    // rewriting them, so that the child can validate and strip its own name.
    static Result<ChildAccessPaths> select_struct_paths_for_child(
            const TColumnAccessPaths& all_paths, const TColumnAccessPaths& predicate_paths,
            const std::string& child_name, bool include_all_paths) {
        ChildAccessPaths child_paths;
        auto select_matching_paths = [&](const TColumnAccessPaths& source_paths,
                                         TColumnAccessPaths& child_paths) -> Status {
            for (const auto& path : source_paths) {
                const bool matches_child = DORIS_TRY(selected_payload_head_matches(
                        path, child_name, PathHeadMatchMode::CASE_INSENSITIVE));
                if (matches_child) {
                    child_paths.emplace_back(path);
                }
            }
            return Status::OK();
        };

        if (include_all_paths) {
            auto status = select_matching_paths(all_paths, child_paths.all_paths);
            if (!status.ok()) {
                return ResultError(std::move(status));
            }
        }
        auto status = select_matching_paths(predicate_paths, child_paths.predicate_paths);
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        return child_paths;
    }

private:
    enum class PathHeadMatchMode { EXACT, CASE_INSENSITIVE };
    enum class MapSelector { UNRECOGNIZED, WILDCARD, KEYS, VALUES };

    // TColumnAccessPath may carry both payload fields after compatibility forwarding. Selected
    // means data_access_path for DATA and meta_access_path for META. Visit only that payload and
    // leave the other payload untouched.
    template <typename AccessPath, typename Visitor>
    static Status visit_selected_payload(AccessPath& access_path, Visitor&& visitor) {
        switch (access_path.type) {
        case TAccessPathType::DATA:
            if (!access_path.__isset.data_access_path) {
                return Status::InternalError(
                        "Invalid DATA access path: data_access_path payload is not set");
            }
            return std::forward<Visitor>(visitor)(access_path.data_access_path);
        case TAccessPathType::META:
            if (!access_path.__isset.meta_access_path) {
                return Status::InternalError(
                        "Invalid META access path: meta_access_path payload is not set");
            }
            return std::forward<Visitor>(visitor)(access_path.meta_access_path);
        default:
            return Status::InternalError("Invalid access path type: {}",
                                         static_cast<int>(access_path.type));
        }
    }

    static Result<bool> selected_payload_head_matches(const TColumnAccessPath& access_path,
                                                      const std::string& expected_head,
                                                      PathHeadMatchMode match_mode) {
        bool matches = false;
        auto status = visit_selected_payload(access_path, [&](const auto& payload) {
            DORIS_CHECK(!payload.path.empty());
            matches = match_mode == PathHeadMatchMode::EXACT
                              ? payload.path.front() == expected_head
                              : StringCaseEqual()(payload.path.front(), expected_head);
            return Status::OK();
        });
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        return matches;
    }

    static Status replace_selected_payload_head(TColumnAccessPath& access_path,
                                                const std::string& child_name) {
        return visit_selected_payload(access_path, [&](auto& payload) {
            DORIS_CHECK(!payload.path.empty());
            payload.path.front() = child_name;
            return Status::OK();
        });
    }

    // Map `*` applies trailing qualifiers only to values, while locating entries still requires
    // complete KEYS as DATA. Construct a fresh key path because the source may be META, and
    // preserve its version so child routing keeps the same legacy/typed encoding.
    static TColumnAccessPath make_full_data_path_for_map_keys(
            const TColumnAccessPath& version_source) {
        TColumnAccessPath child_path;
        child_path.__set_type(TAccessPathType::DATA);
        TDataAccessPath data_path;
        data_path.__set_path({ColumnIterator::ACCESS_MAP_KEYS});
        child_path.__set_data_access_path(data_path);
        if (version_source.__isset.version) {
            child_path.__set_version(version_source.version);
        }
        return child_path;
    }

    static Result<MapSelector> classify_map_selector(const TColumnAccessPath& access_path) {
        MapSelector selector = MapSelector::UNRECOGNIZED;
        auto status = visit_selected_payload(access_path, [&](const auto& payload) {
            DORIS_CHECK(!payload.path.empty());
            const auto& head = payload.path.front();
            if (head == ColumnIterator::ACCESS_ALL) {
                selector = MapSelector::WILDCARD;
            } else if (head == ColumnIterator::ACCESS_MAP_KEYS) {
                selector = MapSelector::KEYS;
            } else if (head == ColumnIterator::ACCESS_MAP_VALUES) {
                selector = MapSelector::VALUES;
            }
            return Status::OK();
        });
        if (!status.ok()) {
            return ResultError(std::move(status));
        }
        return selector;
    }

    static Status distribute_map_paths(TColumnAccessPaths source_paths,
                                       TColumnAccessPaths& key_paths,
                                       TColumnAccessPaths& value_paths) {
        for (auto& path : source_paths) {
            const auto selector = DORIS_TRY(classify_map_selector(path));
            switch (selector) {
            case MapSelector::WILDCARD:
                // A wildcard needs complete keys for runtime lookup, while any remaining
                // qualifiers apply only to the value. The value keeps its type and version.
                key_paths.emplace_back(make_full_data_path_for_map_keys(path));
                RETURN_IF_ERROR(
                        replace_selected_payload_head(path, ColumnIterator::ACCESS_MAP_VALUES));
                value_paths.emplace_back(std::move(path));
                break;
            case MapSelector::KEYS:
                key_paths.emplace_back(std::move(path));
                break;
            case MapSelector::VALUES:
                value_paths.emplace_back(std::move(path));
                break;
            case MapSelector::UNRECOGNIZED:
                break;
            }
        }
        return Status::OK();
    }
};

} // namespace

Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                                  const io::FileReaderSPtr& file_reader,
                                  std::shared_ptr<ColumnReader>* reader) {
    DCHECK(meta.children_columns_size() == 2 || meta.children_columns_size() == 3);

    std::shared_ptr<ColumnReader> item_reader;
    RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(0),
                                         meta.children_columns(0).num_rows(), file_reader,
                                         &item_reader));

    std::shared_ptr<ColumnReader> offset_reader;
    RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(1),
                                         meta.children_columns(1).num_rows(), file_reader,
                                         &offset_reader));

    std::shared_ptr<ColumnReader> null_reader;
    if (meta.is_nullable()) {
        RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(2),
                                             meta.children_columns(2).num_rows(), file_reader,
                                             &null_reader));
    }

    // The num rows of the array reader equals to the num rows of the length reader.
    uint64_t array_num_rows = meta.children_columns(1).num_rows();
    std::shared_ptr<ColumnReader> array_reader(
            new ColumnReader(opts, meta, array_num_rows, file_reader));
    //  array reader do not need to init
    array_reader->_sub_readers.resize(meta.children_columns_size());
    array_reader->_sub_readers[0] = std::move(item_reader);
    array_reader->_sub_readers[1] = std::move(offset_reader);
    if (meta.is_nullable()) {
        array_reader->_sub_readers[2] = std::move(null_reader);
    }
    array_reader->_meta_type = FieldType::OLAP_FIELD_TYPE_ARRAY;
    *reader = std::move(array_reader);
    return Status::OK();
}

Status ColumnReader::create_map(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                                const io::FileReaderSPtr& file_reader,
                                std::shared_ptr<ColumnReader>* reader) {
    // map reader now has 3 sub readers for key, value, offsets(scalar), null(scala)
    DCHECK(meta.children_columns_size() == 3 || meta.children_columns_size() == 4);
    std::shared_ptr<ColumnReader> key_reader;
    RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(0),
                                         meta.children_columns(0).num_rows(), file_reader,
                                         &key_reader));
    std::shared_ptr<ColumnReader> val_reader;
    RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(1),
                                         meta.children_columns(1).num_rows(), file_reader,
                                         &val_reader));
    std::shared_ptr<ColumnReader> offset_reader;
    RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(2),
                                         meta.children_columns(2).num_rows(), file_reader,
                                         &offset_reader));
    std::shared_ptr<ColumnReader> null_reader;
    if (meta.is_nullable()) {
        RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(3),
                                             meta.children_columns(3).num_rows(), file_reader,
                                             &null_reader));
    }

    // The num rows of the map reader equals to the num rows of the length reader.
    uint64_t map_num_rows = meta.children_columns(2).num_rows();
    std::shared_ptr<ColumnReader> map_reader(
            new ColumnReader(opts, meta, map_num_rows, file_reader));
    map_reader->_sub_readers.resize(meta.children_columns_size());

    map_reader->_sub_readers[0] = std::move(key_reader);
    map_reader->_sub_readers[1] = std::move(val_reader);
    map_reader->_sub_readers[2] = std::move(offset_reader);
    if (meta.is_nullable()) {
        map_reader->_sub_readers[3] = std::move(null_reader);
    }
    map_reader->_meta_type = FieldType::OLAP_FIELD_TYPE_MAP;
    *reader = std::move(map_reader);
    return Status::OK();
}

Status ColumnReader::create_struct(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                                   uint64_t num_rows, const io::FileReaderSPtr& file_reader,
                                   std::shared_ptr<ColumnReader>* reader) {
    // not support empty struct
    DCHECK(meta.children_columns_size() >= 1);
    // create struct column reader
    std::shared_ptr<ColumnReader> struct_reader(
            new ColumnReader(opts, meta, num_rows, file_reader));
    struct_reader->_sub_readers.reserve(meta.children_columns_size());
    // now we support struct column can add the children columns according to the schema-change behavior
    for (int i = 0; i < meta.children_columns_size(); i++) {
        std::shared_ptr<ColumnReader> sub_reader;
        RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(i),
                                             meta.children_columns(i).num_rows(), file_reader,
                                             &sub_reader));
        struct_reader->_sub_readers.push_back(std::move(sub_reader));
    }
    struct_reader->_meta_type = FieldType::OLAP_FIELD_TYPE_STRUCT;
    *reader = std::move(struct_reader);
    return Status::OK();
}

Status ColumnReader::create_agg_state(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                                      uint64_t num_rows, const io::FileReaderSPtr& file_reader,
                                      std::shared_ptr<ColumnReader>* reader) {
    if (!meta.has_function_name()) { // meet old version ColumnMetaPB
        std::shared_ptr<ColumnReader> reader_local(
                new ColumnReader(opts, meta, num_rows, file_reader));
        RETURN_IF_ERROR(reader_local->init(&meta));
        *reader = std::move(reader_local);
        return Status::OK();
    }

    auto data_type = DataTypeFactory::instance().create_data_type(meta);
    const auto* agg_state_type = assert_cast<const DataTypeAggState*>(data_type.get());
    agg_state_type->check_function_compatibility(opts.be_exec_version);
    auto type = agg_state_type->get_serialized_type()->get_primitive_type();

    if (read_as_string(type)) {
        std::shared_ptr<ColumnReader> reader_local(
                new ColumnReader(opts, meta, num_rows, file_reader));
        RETURN_IF_ERROR(reader_local->init(&meta));
        *reader = std::move(reader_local);
        return Status::OK();
    } else if (type == PrimitiveType::TYPE_MAP) {
        return create_map(opts, meta, file_reader, reader);
    } else if (type == PrimitiveType::TYPE_ARRAY) {
        return create_array(opts, meta, file_reader, reader);
    } else if (type == PrimitiveType::TYPE_STRUCT) {
        return create_struct(opts, meta, num_rows, file_reader, reader);
    }

    return Status::InternalError("Not supported type: {}, serialized type: {}",
                                 agg_state_type->get_name(), int(type));
}

bool ColumnReader::is_compaction_reader_type(ReaderType type) {
    return type == ReaderType::READER_BASE_COMPACTION ||
           type == ReaderType::READER_CUMULATIVE_COMPACTION ||
           type == ReaderType::READER_COLD_DATA_COMPACTION ||
           type == ReaderType::READER_SEGMENT_COMPACTION ||
           type == ReaderType::READER_FULL_COMPACTION;
}

Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                            uint64_t num_rows, const io::FileReaderSPtr& file_reader,
                            std::shared_ptr<ColumnReader>* reader) {
    if (opts.const_value.has_value()) {
        *reader = std::make_shared<ConstantColumnReader>(*opts.const_value);
        return Status::OK();
    }
    if (is_scalar_type((FieldType)meta.type())) {
        std::shared_ptr<ColumnReader> reader_local(
                new ColumnReader(opts, meta, num_rows, file_reader));
        RETURN_IF_ERROR(reader_local->init(&meta));
        *reader = std::move(reader_local);
        return Status::OK();
    } else {
        auto type = (FieldType)meta.type();
        switch (type) {
        case FieldType::OLAP_FIELD_TYPE_AGG_STATE: {
            return create_agg_state(opts, meta, num_rows, file_reader, reader);
        }
        case FieldType::OLAP_FIELD_TYPE_STRUCT: {
            return create_struct(opts, meta, num_rows, file_reader, reader);
        }
        case FieldType::OLAP_FIELD_TYPE_ARRAY: {
            return create_array(opts, meta, file_reader, reader);
        }
        case FieldType::OLAP_FIELD_TYPE_MAP: {
            return create_map(opts, meta, file_reader, reader);
        }
        case FieldType::OLAP_FIELD_TYPE_VARIANT: {
            // Read variant only root data using a single ColumnReader
            std::shared_ptr<ColumnReader> reader_local(
                    new ColumnReader(opts, meta, num_rows, file_reader));
            RETURN_IF_ERROR(reader_local->init(&meta));
            *reader = std::move(reader_local);
            return Status::OK();
        }
        default:
            return Status::NotSupported("unsupported type for ColumnReader: {}",
                                        std::to_string(int(type)));
        }
    }
}

ColumnReader::ColumnReader() = default;

ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                           uint64_t num_rows, io::FileReaderSPtr file_reader)
        : _use_index_page_cache(!config::disable_storage_page_cache),
          _opts(opts),
          _num_rows(num_rows),
          _file_reader(std::move(file_reader)),
          _dict_encoding_type(UNKNOWN_DICT_ENCODING) {
    _meta_length = meta.length();
    _meta_type = (FieldType)meta.type();
    if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
        _meta_children_column_type = (FieldType)meta.children_columns(0).type();
    }
    _data_type = DataTypeFactory::instance().create_data_type(meta);
    _meta_is_nullable = meta.is_nullable();
    _meta_dict_page = meta.dict_page();
    _meta_compression = meta.compression();
}

ColumnReader::~ColumnReader() = default;

int64_t ColumnReader::get_metadata_size() const {
    return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0);
}

#ifdef BE_TEST
/// This function is only used in UT to verify the correctness of data read from zone map
/// See UT case 'SegCompactionMoWTest.SegCompactionInterleaveWithBig_ooooOOoOooooooooO'
/// be/test/olap/segcompaction_mow_test.cpp
void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) const {
    if (!_segment_zone_map) {
        return;
    }

    const auto rows = dst->size();
    if (rows == 0) {
        return;
    }

    FieldType type = _type;

    if (type != FieldType::OLAP_FIELD_TYPE_INT) {
        return;
    }

    auto* non_nullable_column =
            is_column_nullable(*dst)
                    ? assert_cast<ColumnNullable*>(dst.get())->get_nested_column_ptr().get()
                    : dst.get();

    /// Only verify when the destination column carries Field-accessible TYPE_INT data.
    if (check_and_get_column<ColumnVector<TYPE_INT>>(non_nullable_column) == nullptr) {
        return;
    }

    ZoneMap zone_map;
    THROW_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));

    if (zone_map.has_null) {
        return;
    }

    for (size_t i = 0; i != rows; ++i) {
        Field field;
        dst->get(i, field);
        DCHECK(!field.is_null());
        const auto v = field.get<TYPE_INT>();
        DCHECK_GE(v, zone_map.min_value.get<TYPE_INT>());
        DCHECK_LE(v, zone_map.max_value.get<TYPE_INT>());
    }
}
#endif

Status ColumnReader::init(const ColumnMetaPB* meta) {
    _type = (FieldType)meta->type();

    if (meta->has_be_exec_version()) {
        _be_exec_version = meta->be_exec_version();
    }

    if (_type == FieldType::OLAP_FIELD_TYPE_NONE || _type == FieldType::OLAP_FIELD_TYPE_UNKNOWN) {
        return Status::NotSupported("unsupported typeinfo, type={}", meta->type());
    }
    RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), &_encoding_info));

    for (int i = 0; i < meta->indexes_size(); i++) {
        const auto& index_meta = meta->indexes(i);
        switch (index_meta.type()) {
        case BITMAP_INDEX:
            break;
        case ORDINAL_INDEX:
            _ordinal_index.reset(
                    new OrdinalIndexReader(_file_reader, _num_rows, index_meta.ordinal_index()));
            break;
        case ZONE_MAP_INDEX:
            _segment_zone_map =
                    std::make_unique<ZoneMapPB>(index_meta.zone_map_index().segment_zone_map());
            _zone_map_index.reset(new ZoneMapIndexReader(
                    _file_reader, index_meta.zone_map_index().page_zone_maps()));
            break;
        case BLOOM_FILTER_INDEX:
            _bloom_filter_index.reset(
                    new BloomFilterIndexReader(_file_reader, index_meta.bloom_filter_index()));
            break;
        case NESTED_OFFSETS_INDEX:
            break;
        default:
            return Status::Corruption("Bad file {}: invalid column index type {}",
                                      _file_reader->path().native(), index_meta.type());
        }
    }
    update_metadata_size();

    // ArrayColumnWriter writes a single empty array and flushes. In this scenario,
    // the item writer doesn't write any data and the corresponding ordinal index is empty.
    if (_ordinal_index == nullptr && !is_empty()) {
        return Status::Corruption("Bad file {}: missing ordinal index for column {}",
                                  _file_reader->path().native(), meta->column_id());
    }

    return Status::OK();
}

Status ColumnReader::new_index_iterator(const std::shared_ptr<IndexFileReader>& index_file_reader,
                                        const TabletIndex* index_meta, const std::string& rowset_id,
                                        uint32_t segment_id, size_t rows_of_segment,
                                        std::unique_ptr<IndexIterator>* iterator) {
    RETURN_IF_ERROR(
            _load_index(index_file_reader, index_meta, rowset_id, segment_id, rows_of_segment));
    {
        std::shared_lock<std::shared_mutex> rlock(_load_index_lock);
        auto iter = _index_readers.find(index_meta->index_id());
        if (iter != _index_readers.end()) {
            if (iter->second != nullptr) {
                RETURN_IF_ERROR(iter->second->new_iterator(iterator));
            }
        }
    }
    return Status::OK();
}

Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp,
                               PageHandle* handle, Slice* page_body, PageFooterPB* footer,
                               BlockCompressionCodec* codec) const {
    SCOPED_CONCURRENCY_COUNT(ConcurrencyStatsManager::instance().column_reader_read_page);
    iter_opts.sanity_check();
    PageReadOptions opts(iter_opts.io_ctx);
    opts.verify_checksum = _opts.verify_checksum;
    opts.use_page_cache = iter_opts.use_page_cache;
    opts.kept_in_memory = _opts.kept_in_memory;
    opts.type = iter_opts.type;
    opts.file_reader = iter_opts.file_reader;
    opts.page_pointer = pp;
    opts.codec = codec;
    opts.stats = iter_opts.stats;
    opts.encoding_info = _encoding_info;

    return PageIO::read_and_decompress_page(opts, handle, page_body, footer);
}

Status ColumnReader::get_row_ranges_by_zone_map(
        const AndBlockColumnPredicate* col_predicates,
        const std::vector<std::shared_ptr<const ColumnPredicate>>* delete_predicates,
        RowRanges* row_ranges, const ColumnIteratorOptions& iter_opts) {
    std::vector<uint32_t> page_indexes;
    RETURN_IF_ERROR(
            _get_filtered_pages(col_predicates, delete_predicates, &page_indexes, iter_opts));
    RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges, iter_opts));
    return Status::OK();
}

Status ColumnReader::next_batch_of_zone_map(size_t* n, MutableColumnPtr& dst) const {
    if (_segment_zone_map == nullptr) {
        return Status::InternalError("segment zonemap not exist");
    }
    // TODO: this work to get min/max value seems should only do once
    ZoneMap zone_map;
    RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));

    dst->reserve(*n);
    if (!zone_map.has_not_null) {
        assert_cast<ColumnNullable&>(*dst).insert_many_defaults(*n);
        return Status::OK();
    }
    dst->insert(zone_map.max_value);
    for (int i = 1; i < *n; ++i) {
        dst->insert(zone_map.min_value);
    }
    return Status::OK();
}

Status ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicates,
                                     bool* matched) const {
    *matched = true;
    if (_zone_map_index == nullptr) {
        return Status::OK();
    }
    ZoneMap zone_map;
    RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));

    *matched = _zone_map_match_condition(zone_map, col_predicates);
    return Status::OK();
}

Status ConstantColumnReader::match_condition(const AndBlockColumnPredicate* col_predicates,
                                             bool* matched) const {
    ZoneMap zone_map;
    zone_map.min_value = _value;
    zone_map.max_value = _value;
    zone_map.has_not_null = !_value.is_null();
    // evaluate_and returns false iff no value in [min, max] (i.e. the real constant) can satisfy
    // the predicates; predicates that don't support zonemap conservatively return true.
    *matched = col_predicates->evaluate_and(zone_map);
    return Status::OK();
}

Status ColumnReader::prune_predicates_by_zone_map(
        std::vector<std::shared_ptr<ColumnPredicate>>& predicates, const int column_id,
        bool* pruned) const {
    *pruned = false;
    if (_zone_map_index == nullptr) {
        return Status::OK();
    }

    ZoneMap zone_map;
    RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));
    if (zone_map.pass_all) {
        return Status::OK();
    }

    for (auto it = predicates.begin(); it != predicates.end();) {
        auto predicate = *it;
        if (predicate->column_id() == column_id && predicate->is_always_true(zone_map)) {
            *pruned = true;
            it = predicates.erase(it);
        } else {
            ++it;
        }
    }
    return Status::OK();
}

bool ColumnReader::_zone_map_match_condition(const ZoneMap& zone_map,
                                             const AndBlockColumnPredicate* col_predicates) const {
    if (zone_map.pass_all) {
        return true;
    }

    return col_predicates->evaluate_and(zone_map);
}

Status ColumnReader::_get_filtered_pages(
        const AndBlockColumnPredicate* col_predicates,
        const std::vector<std::shared_ptr<const ColumnPredicate>>* delete_predicates,
        std::vector<uint32_t>* page_indexes, const ColumnIteratorOptions& iter_opts) {
    RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));

    const std::vector<ZoneMapPB>& zone_maps = _zone_map_index->page_zone_maps();
    size_t page_size = _zone_map_index->num_pages();
    for (size_t i = 0; i < page_size; ++i) {
        if (zone_maps[i].pass_all()) {
            page_indexes->push_back(cast_set<uint32_t>(i));
        } else {
            segment_v2::ZoneMap zone_map;
            RETURN_IF_ERROR(ZoneMap::from_proto(zone_maps[i], _data_type, zone_map));
            if (_zone_map_match_condition(zone_map, col_predicates)) {
                bool should_read = true;
                if (delete_predicates != nullptr) {
                    for (auto del_pred : *delete_predicates) {
                        // TODO: Both `min_value` and `max_value` should be 0 or neither should be 0.
                        //  So nullable only need to judge once.
                        if (del_pred->evaluate_del(zone_map)) {
                            should_read = false;
                            break;
                        }
                    }
                }
                if (should_read) {
                    page_indexes->push_back(cast_set<uint32_t>(i));
                }
            }
        }
    }
    VLOG(1) << "total-pages: " << page_size << " not-filtered-pages: " << page_indexes->size()
            << " filtered-percent:"
            << 1.0 - (static_cast<double>(page_indexes->size()) /
                      (static_cast<double>(page_size) * 1.0));
    return Status::OK();
}

Status ColumnReader::_calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
                                           RowRanges* row_ranges,
                                           const ColumnIteratorOptions& iter_opts) {
    row_ranges->clear();
    RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    for (auto i : page_indexes) {
        ordinal_t page_first_id = _ordinal_index->get_first_ordinal(i);
        ordinal_t page_last_id = _ordinal_index->get_last_ordinal(i);
        RowRanges page_row_ranges(RowRanges::create_single(page_first_id, page_last_id + 1));
        RowRanges::ranges_union(*row_ranges, page_row_ranges, row_ranges);
    }
    return Status::OK();
}

Status ColumnReader::get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate* col_predicates,
                                                    RowRanges* row_ranges,
                                                    const ColumnIteratorOptions& iter_opts) {
    RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    RETURN_IF_ERROR(
            _load_bloom_filter_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    RowRanges bf_row_ranges;
    std::unique_ptr<BloomFilterIndexIterator> bf_iter;
    RETURN_IF_ERROR(
            _bloom_filter_index->new_iterator(&bf_iter, iter_opts.stats, &iter_opts.io_ctx));
    size_t range_size = row_ranges->range_size();
    // get covered page ids
    std::set<uint32_t> page_ids;
    for (int i = 0; i < range_size; ++i) {
        int64_t from = row_ranges->get_range_from(i);
        int64_t idx = from;
        int64_t to = row_ranges->get_range_to(i);
        auto iter = _ordinal_index->seek_at_or_before(from);
        while (idx < to && iter.valid()) {
            page_ids.insert(iter.page_index());
            idx = iter.last_ordinal() + 1;
            iter.next();
        }
    }
    for (auto& pid : page_ids) {
        std::unique_ptr<BloomFilter> bf;
        RETURN_IF_ERROR(bf_iter->read_bloom_filter(pid, &bf));
        if (col_predicates->evaluate_and(bf.get())) {
            bf_row_ranges.add(RowRange(_ordinal_index->get_first_ordinal(pid),
                                       _ordinal_index->get_last_ordinal(pid) + 1));
        }
    }
    RowRanges::ranges_intersection(*row_ranges, bf_row_ranges, row_ranges);
    return Status::OK();
}

Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool kept_in_memory,
                                         const ColumnIteratorOptions& iter_opts) {
    if (!_ordinal_index) {
        return Status::InternalError("ordinal_index not inited");
    }
    return _ordinal_index->load(use_page_cache, kept_in_memory, iter_opts.stats, &iter_opts.io_ctx);
}

Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memory,
                                          const ColumnIteratorOptions& iter_opts) {
    if (_zone_map_index != nullptr) {
        return _zone_map_index->load(use_page_cache, kept_in_memory, iter_opts.stats,
                                     &iter_opts.io_ctx);
    }
    return Status::OK();
}

Status ColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const {
    DORIS_CHECK(zone_map != nullptr);
    DORIS_CHECK(_segment_zone_map != nullptr);
    return ZoneMap::from_proto(*_segment_zone_map, _data_type, *zone_map);
}

Status ConstantColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const {
    zone_map->min_value = _value;
    zone_map->max_value = _value;
    zone_map->has_not_null = !_value.is_null();
    return Status::OK();
}

Status ColumnReader::get_page_zone_maps(const ColumnIteratorOptions& iter_opts,
                                        const std::vector<ZoneMapPB>** zone_maps) {
    DORIS_CHECK(zone_maps != nullptr);
    if (_zone_map_index == nullptr) {
        *zone_maps = nullptr;
        return Status::OK();
    }
    RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    *zone_maps = &_zone_map_index->page_zone_maps();
    return Status::OK();
}

Status ColumnReader::get_row_range_for_page(uint32_t page_index,
                                            const ColumnIteratorOptions& iter_opts,
                                            RowRange* row_range) {
    DORIS_CHECK(row_range != nullptr);
    RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    DORIS_CHECK(page_index < _ordinal_index->num_data_pages());
    *row_range = RowRange(_ordinal_index->get_first_ordinal(page_index),
                          _ordinal_index->get_last_ordinal(page_index) + 1);
    return Status::OK();
}

Status ColumnReader::_load_index(const std::shared_ptr<IndexFileReader>& index_file_reader,
                                 const TabletIndex* index_meta, const std::string& rowset_id,
                                 uint32_t segment_id, size_t rows_of_segment) {
    std::unique_lock<std::shared_mutex> wlock(_load_index_lock);

    if (index_meta == nullptr) {
        return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                "Failed to load inverted index: index metadata is null");
    }

    auto it = _index_readers.find(index_meta->index_id());
    if (it != _index_readers.end()) {
        return Status::OK();
    }

    bool should_analyzer =
            inverted_index::InvertedIndexAnalyzer::should_analyzer(index_meta->properties());

    FieldType type;
    if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
        type = _meta_children_column_type;
    } else {
        type = _type;
    }

    if (index_meta->index_type() == IndexType::ANN) {
        _index_readers[index_meta->index_id()] = std::make_shared<AnnIndexReader>(
                index_meta, index_file_reader, rowset_id, segment_id, rows_of_segment);
        return Status::OK();
    }

    IndexReaderPtr index_reader;

    if (is_string_type(type)) {
        if (should_analyzer) {
            try {
                index_reader = FullTextIndexReader::create_shared(index_meta, index_file_reader);
            } catch (const CLuceneError& e) {
                return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                        "create FullTextIndexReader error: {}", e.what());
            }
        } else {
            try {
                index_reader =
                        StringTypeInvertedIndexReader::create_shared(index_meta, index_file_reader);
            } catch (const CLuceneError& e) {
                return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                        "create StringTypeInvertedIndexReader error: {}", e.what());
            }
        }
    } else if (field_is_numeric_type(type)) {
        try {
            index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
        } catch (const CLuceneError& e) {
            return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                    "create BkdIndexReader error: {}", e.what());
        }
    } else {
        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
                "Field type {} is not supported for inverted index", type);
    }
    _index_readers[index_meta->index_id()] = index_reader;
    return Status::OK();
}

bool ColumnReader::has_bloom_filter_index(bool ngram) const {
    if (_bloom_filter_index == nullptr) return false;

    if (ngram) {
        return _bloom_filter_index->algorithm() == BloomFilterAlgorithmPB::NGRAM_BLOOM_FILTER;
    } else {
        return _bloom_filter_index->algorithm() != BloomFilterAlgorithmPB::NGRAM_BLOOM_FILTER;
    }
}

Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool kept_in_memory,
                                              const ColumnIteratorOptions& iter_opts) {
    if (_bloom_filter_index != nullptr) {
        return _bloom_filter_index->load(use_page_cache, kept_in_memory, iter_opts.stats,
                                         &iter_opts.io_ctx);
    }
    return Status::OK();
}

Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter,
                                       const ColumnIteratorOptions& iter_opts) {
    RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
    *iter = _ordinal_index->seek_at_or_before(ordinal);
    if (!iter->valid()) {
        return Status::NotFound("Failed to seek to ordinal {}, ", ordinal);
    }
    return Status::OK();
}

Status ColumnReader::get_ordinal_index_reader(OrdinalIndexReader*& reader,
                                              OlapReaderStatistics* index_load_stats) {
    CHECK(_ordinal_index) << fmt::format("ordinal index is null for column reader of type {}",
                                         std::to_string(int(_meta_type)));
    RETURN_IF_ERROR(
            _ordinal_index->load(_use_index_page_cache, _opts.kept_in_memory, index_load_stats));
    reader = _ordinal_index.get();
    return Status::OK();
}

Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColumn* tablet_column) {
    return new_iterator(iterator, tablet_column, nullptr);
}

Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColumn* tablet_column,
                                  const StorageReadOptions* opt) {
    if (is_empty()) {
        *iterator = std::make_unique<EmptyFileColumnIterator>();
        return Status::OK();
    }
    if (is_scalar_type(_meta_type)) {
        if (is_string_type(_meta_type)) {
            *iterator = std::make_unique<StringFileColumnIterator>(shared_from_this());
        } else {
            *iterator = std::make_unique<FileColumnIterator>(shared_from_this());
        }
        (*iterator)->set_column_name(tablet_column ? tablet_column->name() : "");
        return Status::OK();
    } else {
        auto type = _meta_type;
        switch (type) {
        case FieldType::OLAP_FIELD_TYPE_AGG_STATE: {
            return new_agg_state_iterator(iterator);
        }
        case FieldType::OLAP_FIELD_TYPE_STRUCT: {
            return new_struct_iterator(iterator, tablet_column);
        }
        case FieldType::OLAP_FIELD_TYPE_ARRAY: {
            return new_array_iterator(iterator, tablet_column);
        }
        case FieldType::OLAP_FIELD_TYPE_MAP: {
            return new_map_iterator(iterator, tablet_column);
        }
        default:
            return Status::NotSupported("unsupported type to create iterator: {}",
                                        std::to_string(int(type)));
        }
    }
}

Status ColumnReader::new_agg_state_iterator(ColumnIteratorUPtr* iterator) {
    *iterator = std::make_unique<FileColumnIterator>(shared_from_this());
    return Status::OK();
}

Status ColumnReader::new_array_iterator(ColumnIteratorUPtr* iterator,
                                        const TabletColumn* tablet_column) {
    ColumnIteratorUPtr item_iterator;
    RETURN_IF_ERROR(_sub_readers[0]->new_iterator(
            &item_iterator, tablet_column && tablet_column->get_subtype_count() > 0
                                    ? &tablet_column->get_sub_column(0)
                                    : nullptr));

    item_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");

    ColumnIteratorUPtr offset_iterator;
    RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator, nullptr));
    auto* file_iter = static_cast<FileColumnIterator*>(offset_iterator.release());
    OffsetFileColumnIteratorUPtr ofcIter = std::make_unique<OffsetFileColumnIterator>(
            std::unique_ptr<FileColumnIterator>(file_iter));

    ColumnIteratorUPtr null_iterator;
    if (is_nullable()) {
        RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator, nullptr));
    }
    *iterator = std::make_unique<ArrayFileColumnIterator>(shared_from_this(), std::move(ofcIter),
                                                          std::move(item_iterator),
                                                          std::move(null_iterator));
    return Status::OK();
}

Status ColumnReader::new_map_iterator(ColumnIteratorUPtr* iterator,
                                      const TabletColumn* tablet_column) {
    ColumnIteratorUPtr key_iterator;
    RETURN_IF_ERROR(_sub_readers[0]->new_iterator(
            &key_iterator, tablet_column && tablet_column->get_subtype_count() > 1
                                   ? &tablet_column->get_sub_column(0)
                                   : nullptr));
    ColumnIteratorUPtr val_iterator;
    RETURN_IF_ERROR(_sub_readers[1]->new_iterator(
            &val_iterator, tablet_column && tablet_column->get_subtype_count() > 1
                                   ? &tablet_column->get_sub_column(1)
                                   : nullptr));
    ColumnIteratorUPtr offsets_iterator;
    RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr));
    auto* file_iter = static_cast<FileColumnIterator*>(offsets_iterator.release());
    OffsetFileColumnIteratorUPtr ofcIter = std::make_unique<OffsetFileColumnIterator>(
            std::unique_ptr<FileColumnIterator>(file_iter));

    ColumnIteratorUPtr null_iterator;
    if (is_nullable()) {
        RETURN_IF_ERROR(_sub_readers[3]->new_iterator(&null_iterator, nullptr));
    }
    *iterator = std::make_unique<MapFileColumnIterator>(
            shared_from_this(), std::move(null_iterator), std::move(ofcIter),
            std::move(key_iterator), std::move(val_iterator));
    return Status::OK();
}

Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
                                         const TabletColumn* tablet_column) {
    std::vector<ColumnIteratorUPtr> sub_column_iterators;
    size_t child_size = is_nullable() ? _sub_readers.size() - 1 : _sub_readers.size();
    size_t tablet_column_size = tablet_column ? tablet_column->get_sub_columns().size() : 0;
    sub_column_iterators.reserve(child_size);

    for (uint64_t i = 0; i < child_size; i++) {
        ColumnIteratorUPtr sub_column_iterator;
        RETURN_IF_ERROR(_sub_readers[i]->new_iterator(
                &sub_column_iterator, tablet_column ? &tablet_column->get_sub_column(i) : nullptr));
        sub_column_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(i).name()
                                                           : "");
        sub_column_iterators.emplace_back(std::move(sub_column_iterator));
    }

    // create default_iterator for schema-change behavior which increase column
    for (size_t i = child_size; i < tablet_column_size; i++) {
        TabletColumn column = tablet_column->get_sub_column(i);
        ColumnIteratorUPtr it;
        RETURN_IF_ERROR(Segment::new_default_iterator(column, &it));
        it->set_column_name(column.name());
        sub_column_iterators.emplace_back(std::move(it));
    }

    ColumnIteratorUPtr null_iterator;
    if (is_nullable()) {
        RETURN_IF_ERROR(_sub_readers[child_size]->new_iterator(&null_iterator, nullptr));
    }
    *iterator = std::make_unique<StructFileColumnIterator>(
            shared_from_this(), std::move(null_iterator), std::move(sub_column_iterators));
    return Status::OK();
}

void ColumnIterator::_convert_to_place_holder_column(MutableColumnPtr& dst, size_t count) {
    if (_read_phase == ReadPhase::LAZY) {
        return;
    } else if (_read_requirement == ReadRequirement::LAZY_OUTPUT &&
               _read_phase == ReadPhase::PREDICATE) {
        // This branch is for non-predicate columns that still have to appear in the
        // predicate-phase block so row filtering can keep all block columns aligned.
        // Columns already marked PREDICATE are read normally, and SKIP/NORMAL
        // columns do not participate in lazy materialization.
        _has_place_holder_column = true;
    }

    dst->insert_many_defaults(count);
}

void ColumnIterator::_recovery_from_place_holder_column(MutableColumnPtr& dst) {
    if (_read_phase == ReadPhase::LAZY && _has_place_holder_column) {
        dst->clear();
        _has_place_holder_column = false;
    }
}

Result<ColumnIterator::AccessPathSplit> ColumnIterator::_split_access_paths(
        TColumnAccessPaths access_paths) const {
    AccessPathSplit split;
    for (auto& path : access_paths) {
        const bool uses_legacy_encoding = uses_legacy_access_path_encoding(path);
        if (!uses_legacy_encoding &&
            path.version != g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED) {
            return ResultError(
                    Status::InternalError("Unsupported access path version: {}", path.version));
        }

        std::vector<std::string>* components = nullptr;
        if (uses_legacy_encoding) {
            if (path.type != TAccessPathType::DATA) {
                return ResultError(Status::InternalError("Invalid legacy access path type: {}",
                                                         static_cast<int>(path.type)));
            }
            if (!path.__isset.data_access_path) {
                return ResultError(Status::InternalError(
                        "Invalid legacy access path: data_access_path payload is not set"));
            }
            components = &path.data_access_path.path;
        } else {
            switch (path.type) {
            case TAccessPathType::DATA:
                if (!path.__isset.data_access_path) {
                    return ResultError(Status::InternalError(
                            "Invalid DATA access path: data_access_path payload is not set"));
                }
                components = &path.data_access_path.path;
                break;
            case TAccessPathType::META:
                if (!path.__isset.meta_access_path) {
                    return ResultError(Status::InternalError(
                            "Invalid META access path: meta_access_path payload is not set"));
                }
                components = &path.meta_access_path.path;
                break;
            default:
                return ResultError(Status::InternalError("Invalid access path type: {}",
                                                         static_cast<int>(path.type)));
            }
        }

        if (components->empty()) {
            return ResultError(Status::InternalError(
                    "Invalid access path for column '{}': path is empty", _column_name));
        }

        if (!StringCaseEqual()((*components)[0], _column_name)) {
            return ResultError(Status::InternalError(
                    R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
                    (*components)[0]));
        }

        components->erase(components->begin());
        if (components->empty()) {
            split.reads_current_data = true;
            continue;
        }

        const bool is_current_level_meta =
                components->size() == 1 && is_meta_access_path_component((*components)[0]) &&
                (path.type == TAccessPathType::META || uses_legacy_encoding);
        if (is_current_level_meta) {
            if (StringCaseEqual()((*components)[0], ACCESS_OFFSET)) {
                split.current_meta_mode = MetaReadMode::OFFSET_ONLY;
            } else if (split.current_meta_mode == MetaReadMode::DEFAULT) {
                split.current_meta_mode = MetaReadMode::NULL_MAP_ONLY;
            }
            continue;
        }

        split.descendant_paths.emplace_back(std::move(path));
    }
    return split;
}

Result<ColumnIterator::NestedAccessPathPlan> ColumnIterator::_prepare_nested_access_paths(
        const TColumnAccessPaths& all_access_paths,
        const TColumnAccessPaths& predicate_access_paths, NestedMetaSupport meta_support,
        const std::function<void(ReadRequirement)>& set_all_data_descendants_read_requirement) {
    const auto requirement_before = _read_requirement;
    if (!predicate_access_paths.empty()) {
        set_read_requirement_self(ReadRequirement::PREDICATE);
    }

    NestedAccessPathPlan plan;
    auto all_split = _split_access_paths(all_access_paths);
    if (!all_split.has_value()) {
        return ResultError(std::move(all_split).error());
    }
    plan.all = std::move(all_split).value();

    auto predicate_split = _split_access_paths(predicate_access_paths);
    if (!predicate_split.has_value()) {
        return ResultError(std::move(predicate_split).error());
    }
    plan.predicate = std::move(predicate_split).value();
    if (plan.all.reads_current_data) {
        set_lazy_output_requirement();
    }
    if (plan.predicate.reads_current_data) {
        set_read_requirement(ReadRequirement::PREDICATE);
    }

    if (!plan.predicate.has_descendant_paths()) {
        RETURN_IF_ERROR_RESULT(_check_and_set_meta_read_mode(requirement_before, plan.all));
        plan.skip_data_descendants =
                read_null_map_only() ||
                (meta_support == NestedMetaSupport::NULL_MAP_AND_OFFSET && read_offset_only());
        if (plan.skip_data_descendants) {
            set_all_data_descendants_read_requirement(ReadRequirement::SKIP);
        }
    }
    return plan;
}

///====================== MapFileColumnIterator ============================////
MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reader,
                                             ColumnIteratorUPtr null_iterator,
                                             OffsetFileColumnIteratorUPtr offsets_iterator,
                                             ColumnIteratorUPtr key_iterator,
                                             ColumnIteratorUPtr val_iterator)
        : _map_reader(reader),
          _offsets_iterator(std::move(offsets_iterator)),
          _key_iterator(std::move(key_iterator)),
          _val_iterator(std::move(val_iterator)) {
    if (_map_reader->is_nullable()) {
        _null_iterator = std::move(null_iterator);
    }
    // Access paths identify map children by logical selectors rather than storage/schema child
    // names. These names are consumed by each child's _split_access_paths().
    _key_iterator->set_column_name(ACCESS_MAP_KEYS);
    _val_iterator->set_column_name(ACCESS_MAP_VALUES);
}

Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
    if (_read_requirement == ReadRequirement::SKIP) {
        DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }
    RETURN_IF_ERROR(_key_iterator->init(opts));
    RETURN_IF_ERROR(_val_iterator->init(opts));
    RETURN_IF_ERROR(_offsets_iterator->init(opts));
    if (_map_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init(opts));
    }
    return Status::OK();
}

Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
    if (!need_to_read()) {
        DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    if (read_null_map_only()) {
        // In NULL_MAP_ONLY mode, only seek the null iterator; skip offset/key/val iterators
        if (_map_reader->is_nullable() && _null_iterator) {
            RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
        }
        return Status::OK();
    }

    if (_map_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
    }
    RETURN_IF_ERROR(_offsets_iterator->seek_to_ordinal(ord));
    if (read_offset_only()) {
        // In OFFSET_ONLY mode, key/value iterators are SKIP, no need to seek them
        return Status::OK();
    }
    // here to use offset info
    ordinal_t offset = 0;
    RETURN_IF_ERROR(_offsets_iterator->_peek_one_offset(&offset));
    RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(offset));
    RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(offset));
    return Status::OK();
}

Status MapFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& params) {
    RETURN_IF_ERROR(_offsets_iterator->init_prefetcher(params));
    if (_map_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init_prefetcher(params));
    }
    RETURN_IF_ERROR(_key_iterator->init_prefetcher(params));
    RETURN_IF_ERROR(_val_iterator->init_prefetcher(params));
    return Status::OK();
}

void MapFileColumnIterator::collect_prefetchers(
        std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>& prefetchers,
        PrefetcherInitMethod init_method) {
    if (!need_to_read()) {
        return;
    }
    if (!read_null_map_only()) {
        _offsets_iterator->collect_prefetchers(prefetchers, init_method);
    }
    if (_map_reader->is_nullable()) {
        _null_iterator->collect_prefetchers(prefetchers, init_method);
    }
    if (read_offset_only() || read_null_map_only()) {
        return;
    }
    // the actual data pages to read of key/value column depends on the read result of offset column,
    // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here.
    if (_key_iterator->need_to_read()) {
        _key_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS);
    }
    if (_val_iterator->need_to_read()) {
        _val_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS);
    }
}

Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    if (!need_to_read()) {
        DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, *n);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        // NULL_MAP_ONLY mode: read null map, fill nested ColumnMap with empty defaults
        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        auto null_map_ptr = nullable_col.get_null_map_column_ptr();
        size_t num_read = *n;
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            // schema-change: column became nullable but old segment has no null data
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
        // fill nested ColumnMap with empty (zero-element) maps
        auto& column_map = assert_cast<ColumnMap&, TypeCheckOnRelease::DISABLE>(
                nullable_col.get_nested_column());
        column_map.insert_many_defaults(num_read);
        *has_null = true;
        return Status::OK();
    }

    auto& column_map = assert_cast<ColumnMap&, TypeCheckOnRelease::DISABLE>(
            is_column_nullable(*dst) ? static_cast<ColumnNullable&>(*dst).get_nested_column()
                                     : *dst);
    const bool read_meta_columns = need_to_read_meta_columns();
    MutableColumnPtr column_offsets_ptr;
    if (read_meta_columns) {
        column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr()));
    } else {
        // The parent offsets were already materialized in the predicate phase, so
        // they must not be appended to dst again. We still read offsets into a
        // temporary column here: this sequential path may be serving a nested
        // lazy read after seek_to_ordinal(), and the storage offsets are needed to
        // compute how many key/value elements to read from the current source
        // ordinal. The existing dst offsets only describe the filtered output
        // shape and do not track the current source ordinal consumed by this
        // iterator call.
        const auto base_offset =
                column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back();
        column_offsets_ptr = ColumnMap::COffsets::create();
        assert_cast<ColumnOffset64&, TypeCheckOnRelease::DISABLE>(*column_offsets_ptr)
                .insert_value(base_offset);
    }
    Defer defer_offsets {[&] {
        if (read_meta_columns) {
            auto typed_column_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr(
                    assert_cast<ColumnMap::COffsets*, TypeCheckOnRelease::DISABLE>(
                            column_offsets_ptr.get()));
            column_offsets_ptr = nullptr;
            column_map.get_offsets_ptr() = std::move(typed_column_offsets_ptr);
        }
    }};
    bool offsets_has_null = false;
    ssize_t start = column_offsets_ptr->size();
    RETURN_IF_ERROR(_offsets_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null));
    if (*n == 0) {
        return Status::OK();
    }
    auto& column_offsets = static_cast<ColumnArray::ColumnOffsets&>(*column_offsets_ptr);
    RETURN_IF_ERROR(_offsets_iterator->_calculate_offsets(start, column_offsets));
    DCHECK(column_offsets.get_data().back() >= column_offsets.get_data()[start - 1]);
    size_t num_items =
            column_offsets.get_data().back() - column_offsets.get_data()[start - 1]; // -1 is valid

    if (num_items > 0) {
        auto key_ptr = IColumn::mutate(std::move(column_map.get_keys_ptr()));
        auto val_ptr = IColumn::mutate(std::move(column_map.get_values_ptr()));
        Defer defer_keys {[&] { column_map.get_keys_ptr() = std::move(key_ptr); }};
        Defer defer_values {[&] { column_map.get_values_ptr() = std::move(val_ptr); }};
        if (read_offset_only()) {
            // OFFSET_ONLY mode: skip reading actual key/value data, fill with defaults
            key_ptr->insert_many_defaults(num_items);
            val_ptr->insert_many_defaults(num_items);
        } else {
            auto read_or_fill_child = [&](ColumnIterator* iterator,
                                          MutableColumnPtr& column) -> Status {
                if (_read_phase == ReadPhase::LAZY && read_meta_columns &&
                    iterator->read_requirement() == ReadRequirement::SKIP) {
                    column->insert_many_defaults(num_items);
                    return Status::OK();
                }

                bool dummy_has_null = false;
                size_t num_read = num_items;
                RETURN_IF_ERROR(iterator->next_batch(&num_read, column, &dummy_has_null));
                DCHECK(num_read == num_items);
                return Status::OK();
            };
            RETURN_IF_ERROR(read_or_fill_child(_key_iterator.get(), key_ptr));
            RETURN_IF_ERROR(read_or_fill_child(_val_iterator.get(), val_ptr));
        }
    }

    if (is_column_nullable(*dst) && read_meta_columns) {
        size_t num_read = *n;
        auto null_map_ptr = static_cast<ColumnNullable&>(*dst).get_null_map_column_ptr();
        // in not-null to null linked-schemachange mode,
        // actually we do not change dat data include meta in footer,
        // so may dst from changed meta which is nullable but old data is not nullable,
        // if so, we should set null_map to all null by default
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
    }
    return Status::OK();
}

Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
                                             MutableColumnPtr& dst) {
    if (!need_to_read()) {
        DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, count);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        // NULL_MAP_ONLY mode: read null map by rowids, fill nested ColumnMap with empty defaults
        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        if (_null_iterator) {
            auto null_map_ptr = nullable_col.get_null_map_column_ptr();
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_column));
        } else {
            // schema-change: column became nullable but old segment has no null data
            auto null_map_ptr = nullable_col.get_null_map_column_ptr();
            null_map_ptr->insert_many_vals(0, count);
        }
        // fill nested ColumnMap with empty (zero-element) maps
        auto& column_map = assert_cast<ColumnMap&, TypeCheckOnRelease::DISABLE>(
                nullable_col.get_nested_column());
        column_map.insert_many_defaults(count);
        return Status::OK();
    }

    if (count == 0) {
        return Status::OK();
    }

    // resolve ColumnMap and nullable wrapper
    auto& column_map = assert_cast<ColumnMap&, TypeCheckOnRelease::DISABLE>(
            is_column_nullable(*dst) ? static_cast<ColumnNullable&>(*dst).get_nested_column()
                                     : *dst);
    const bool read_meta_columns = need_to_read_meta_columns();
    MutableColumnPtr offsets_ptr;
    if (read_meta_columns) {
        offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr()));
    } else {
        const auto base_offset =
                column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back();
        offsets_ptr = ColumnMap::COffsets::create();
        assert_cast<ColumnOffset64&, TypeCheckOnRelease::DISABLE>(*offsets_ptr)
                .insert_value(base_offset);
    }
    Defer defer_offsets {[&] {
        if (read_meta_columns) {
            auto typed_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr(
                    assert_cast<ColumnMap::COffsets*, TypeCheckOnRelease::DISABLE>(
                            offsets_ptr.get()));
            offsets_ptr = nullptr;
            column_map.get_offsets_ptr() = std::move(typed_offsets_ptr);
        }
    }};
    auto& offsets = static_cast<ColumnArray::ColumnOffsets&>(*offsets_ptr);
    size_t base = offsets.get_data().empty() ? 0 : offsets.get_data().back();

    // 1. bulk read null-map if nullable
    std::vector<uint8_t> null_mask; // 0: not null, 1: null
    if (read_meta_columns) {
        if (_map_reader->is_nullable()) {
            // For nullable map columns, the destination column must also be nullable.
            if (UNLIKELY(!is_column_nullable(*dst))) {
                return Status::InternalError(
                        "unexpected non-nullable destination column for nullable map reader");
            }
            MutableColumnPtr null_map_ptr =
                    static_cast<ColumnNullable&>(*dst).get_null_map_column_ptr();
            size_t null_before = null_map_ptr->size();
            RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_ptr));
            // extract a light-weight view to decide element reads
            auto& null_map_col = assert_cast<ColumnUInt8&>(*null_map_ptr);
            const auto* src = null_map_col.get_data().data() + null_before;
            null_mask.assign(src, src + count);
        } else if (is_column_nullable(*dst)) {
            // in not-null to null linked-schemachange mode,
            // actually we do not change dat data include meta in footer,
            // so may dst from changed meta which is nullable but old data is not nullable,
            // if so, we should set null_map to all null by default
            MutableColumnPtr null_map_ptr =
                    static_cast<ColumnNullable&>(*dst).get_null_map_column_ptr();
            auto& null_map = assert_cast<ColumnUInt8&>(*null_map_ptr);
            null_map.insert_many_vals(0, count);
        }
    } else if (_map_reader->is_nullable()) {
        // In lazy mode the parent null map has already been materialized during
        // predicate read and filtered together with the block. Reuse that dst
        // null map to avoid re-reading the same meta column from storage.
        if (UNLIKELY(!is_column_nullable(*dst))) {
            return Status::InternalError(
                    "unexpected non-nullable destination column for nullable map reader");
        }
        const auto& null_map_col = static_cast<const ColumnNullable&>(*dst).get_null_map_column();
        DORIS_CHECK(null_map_col.size() == count);
        const auto* src = null_map_col.get_data().data();
        null_mask.assign(src, src + count);
    }

    // 2. Bulk read source start ordinals for requested rows. The offsets stored
    // in dst already describe the filtered output shape when read_meta_columns is
    // false, but they do not contain the source key/value ordinal for each
    // selected rowid. We still need the storage offsets here to seek child
    // iterators to the correct source element ranges.
    MutableColumnPtr starts_col = ColumnOffset64::create();
    starts_col->reserve(count);
    RETURN_IF_ERROR(_offsets_iterator->read_by_rowids(rowids, count, starts_col));

    // 3. bulk read next-start ordinals for rowid+1 (within bounds)
    std::vector<rowid_t> next_rowids(count);
    for (size_t i = 0; i < count; ++i) {
        uint64_t nr = rowids[i] + 1;
        next_rowids[i] = nr < _map_reader->num_rows() ? static_cast<rowid_t>(nr)
                                                      : static_cast<rowid_t>(0); // placeholder
    }
    MutableColumnPtr next_starts_col = ColumnOffset64::create();
    next_starts_col->reserve(count);
    // read for all; we'll fix out-of-bound cases below
    RETURN_IF_ERROR(_offsets_iterator->read_by_rowids(next_rowids.data(), count, next_starts_col));

    // 4. fix next_start for rows whose next_rowid is out-of-bound (rowid == num_rows-1)
    for (size_t i = 0; i < count; ++i) {
        if (rowids[i] + 1 >= _map_reader->num_rows()) {
            // seek to the last row and consume one to move decoder to end-of-page,
            // then peek page-tail sentinel next_array_item_ordinal as next_start
            RETURN_IF_ERROR(_offsets_iterator->seek_to_ordinal(rowids[i]));
            size_t one = 1;
            bool has_null_unused = false;
            MutableColumnPtr tmp = ColumnOffset64::create();
            RETURN_IF_ERROR(_offsets_iterator->next_batch(&one, tmp, &has_null_unused));
            ordinal_t ns = 0;
            RETURN_IF_ERROR(_offsets_iterator->_peek_one_offset(&ns));
            // overwrite with sentinel
            assert_cast<ColumnOffset64&, TypeCheckOnRelease::DISABLE>(*next_starts_col)
                    .get_data()[i] = ns;
        }
    }

    // 5. compute sizes and append offsets prefix-sum
    auto& starts_data = assert_cast<ColumnOffset64&>(*starts_col).get_data();
    auto& next_starts_data = assert_cast<ColumnOffset64&>(*next_starts_col).get_data();
    std::vector<size_t> sizes(count, 0);
    size_t acc = base;
    if (read_meta_columns) {
        offsets.get_data().reserve(offsets.get_data().size() + count);
    }
    for (size_t i = 0; i < count; ++i) {
        auto sz = static_cast<size_t>(next_starts_data[i] - starts_data[i]);
        if (_map_reader->is_nullable() && !null_mask.empty() && null_mask[i]) {
            sz = 0; // null rows do not consume elements
        }
        sizes[i] = sz;
        acc += sz;
        if (read_meta_columns) {
            offsets.get_data().push_back(acc);
        }
    }

    // 6. read key/value elements for non-empty sizes
    auto keys_ptr = IColumn::mutate(std::move(column_map.get_keys_ptr()));
    auto vals_ptr = IColumn::mutate(std::move(column_map.get_values_ptr()));
    Defer defer_keys {[&] { column_map.get_keys_ptr() = std::move(keys_ptr); }};
    Defer defer_values {[&] { column_map.get_values_ptr() = std::move(vals_ptr); }};
    // In lazy phase with read_meta_columns=true, this map was only a placeholder during
    // predicate evaluation and is cleared before the lazy read. If only KEYS or VALUES is
    // requested, fill the skipped counterpart with defaults to keep ColumnMap's
    // key/value/offset sizes consistent without reading unnecessary data pages.
    const bool fill_lazy_skipped_keys = _read_phase == ReadPhase::LAZY && read_meta_columns &&
                                        _key_iterator->read_requirement() == ReadRequirement::SKIP;
    const bool fill_lazy_skipped_values =
            _read_phase == ReadPhase::LAZY && read_meta_columns &&
            _val_iterator->read_requirement() == ReadRequirement::SKIP;
    auto read_or_fill_range = [&](ColumnIterator* iterator, MutableColumnPtr& column,
                                  ordinal_t start_idx, size_t num_items,
                                  bool fill_lazy_skipped_child) -> Status {
        if (num_items == 0) {
            return Status::OK();
        }
        if (fill_lazy_skipped_child) {
            column->insert_many_defaults(num_items);
            return Status::OK();
        }
        if (_read_phase == ReadPhase::LAZY && !iterator->need_to_read()) {
            return Status::OK();
        }
        size_t n = num_items;
        bool dummy_has_null = false;
        RETURN_IF_ERROR(iterator->seek_to_ordinal(start_idx));
        RETURN_IF_ERROR(iterator->next_batch(&n, column, &dummy_has_null));
        DCHECK(n == num_items);
        return Status::OK();
    };

    size_t this_run = sizes[0];
    auto start_idx = starts_data[0];
    auto last_idx = starts_data[0] + this_run;
    for (size_t i = 1; i < count; ++i) {
        size_t sz = sizes[i];
        if (sz == 0) {
            continue;
        }
        auto start = static_cast<ordinal_t>(starts_data[i]);
        if (start != last_idx) {
            RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr, start_idx, this_run,
                                               fill_lazy_skipped_keys));
            RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr, start_idx, this_run,
                                               fill_lazy_skipped_values));
            start_idx = start;
            this_run = sz;
            last_idx = start + sz;
            continue;
        }

        this_run += sz;
        last_idx += sz;
    }

    RETURN_IF_ERROR(read_or_fill_range(_key_iterator.get(), keys_ptr, start_idx, this_run,
                                       fill_lazy_skipped_keys));
    RETURN_IF_ERROR(read_or_fill_range(_val_iterator.get(), vals_ptr, start_idx, this_run,
                                       fill_lazy_skipped_values));
    return Status::OK();
}

void MapFileColumnIterator::set_lazy_output_requirement() {
    set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    _key_iterator->set_lazy_output_requirement();
    _val_iterator->set_lazy_output_requirement();
}

void MapFileColumnIterator::remove_pruned_sub_iterators() {
    _key_iterator->remove_pruned_sub_iterators();
    _val_iterator->remove_pruned_sub_iterators();
}

Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
                                               const TColumnAccessPaths& predicate_access_paths) {
    if (all_access_paths.empty() && predicate_access_paths.empty()) {
        return Status::OK();
    }

    auto plan = DORIS_TRY(_prepare_nested_access_paths(
            all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP_AND_OFFSET,
            [this](ReadRequirement requirement) {
                _key_iterator->set_read_requirement(requirement);
                _val_iterator->set_read_requirement(requirement);
            }));
    if (plan.skip_data_descendants) {
        return Status::OK();
    }

    if (!plan.all.has_descendant_paths() && !plan.predicate.has_descendant_paths()) {
        return Status::OK();
    }

    auto child_paths = DORIS_TRY(DescendantAccessPathRouter::route_map_paths_to_children(
            std::move(plan.all.descendant_paths), std::move(plan.predicate.descendant_paths)));

    if (!child_paths.key.empty()) {
        RETURN_IF_ERROR(_key_iterator->set_access_paths(child_paths.key.all_paths,
                                                        child_paths.key.predicate_paths));
        // Apply LAZY_OUTPUT after child predicate paths have been handled. Read requirements are
        // monotonic, so a predicate-only child already promoted to PREDICATE will not
        // be downgraded, while a non-predicate child becomes a lazy materialization target.
        _key_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    } else {
        _key_iterator->set_read_requirement(ReadRequirement::SKIP);
        DLOG(INFO) << "Map column iterator set key column to SKIP";
    }

    if (!child_paths.value.empty()) {
        RETURN_IF_ERROR(_val_iterator->set_access_paths(child_paths.value.all_paths,
                                                        child_paths.value.predicate_paths));
        // Same as keys: predicate-only value paths stay PREDICATE because this
        // post-processing update cannot lower a stronger child requirement.
        _val_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    } else {
        _val_iterator->set_read_requirement(ReadRequirement::SKIP);
        DLOG(INFO) << "Map column iterator set value column to SKIP";
    }
    return Status::OK();
}

void MapFileColumnIterator::set_read_phase(ReadPhase mode) {
    ColumnIterator::set_read_phase(mode);
    _key_iterator->set_read_phase(mode);
    _val_iterator->set_read_phase(mode);
}

void MapFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) {
    _recovery_from_place_holder_column(dst);
    auto& map_column = assert_cast<ColumnMap&, TypeCheckOnRelease::DISABLE>(
            dst->is_nullable() ? static_cast<ColumnNullable&>(*dst).get_nested_column() : *dst);
    auto keys_ptr = IColumn::mutate(std::move(map_column.get_keys_ptr()));
    auto vals_ptr = IColumn::mutate(std::move(map_column.get_values_ptr()));
    _key_iterator->finalize_lazy_phase(keys_ptr);
    _val_iterator->finalize_lazy_phase(vals_ptr);
    map_column.get_keys_ptr() = std::move(keys_ptr);
    map_column.get_values_ptr() = std::move(vals_ptr);
}

void MapFileColumnIterator::set_read_requirement(ReadRequirement requirement) {
    set_read_requirement_self(requirement);
    _key_iterator->set_read_requirement(requirement);
    _val_iterator->set_read_requirement(requirement);
}

bool MapFileColumnIterator::has_lazy_read_target() const {
    return _read_requirement == ReadRequirement::LAZY_OUTPUT ||
           _key_iterator->has_lazy_read_target() || _val_iterator->has_lazy_read_target();
}

////////////////////////////////////////////////////////////////////////////////

StructFileColumnIterator::StructFileColumnIterator(
        std::shared_ptr<ColumnReader> reader, ColumnIteratorUPtr null_iterator,
        std::vector<ColumnIteratorUPtr>&& sub_column_iterators)
        : _struct_reader(reader), _sub_column_iterators(std::move(sub_column_iterators)) {
    if (_struct_reader->is_nullable()) {
        _null_iterator = std::move(null_iterator);
    }
}

Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
    if (_read_requirement == ReadRequirement::SKIP) {
        DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    for (auto& column_iterator : _sub_column_iterators) {
        RETURN_IF_ERROR(column_iterator->init(opts));
    }
    if (_struct_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init(opts));
    }
    return Status::OK();
}

Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    if (!need_to_read()) {
        DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, *n);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        // NULL_MAP_ONLY mode: read null map, fill nested ColumnStruct with empty defaults
        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        auto null_map_ptr = nullable_col.get_null_map_column_ptr();
        size_t num_read = *n;
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            // schema-change: column became nullable but old segment has no null data
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
        // fill nested ColumnStruct with defaults to maintain consistent column sizes
        auto& column_struct = assert_cast<ColumnStruct&, TypeCheckOnRelease::DISABLE>(
                nullable_col.get_nested_column());
        column_struct.insert_many_defaults(num_read);
        *has_null = true;
        return Status::OK();
    }

    auto& column_struct = assert_cast<ColumnStruct&, TypeCheckOnRelease::DISABLE>(
            is_column_nullable(*dst) ? static_cast<ColumnNullable&>(*dst).get_nested_column()
                                     : *dst);
    for (size_t i = 0; i < column_struct.tuple_size(); i++) {
        size_t num_read = *n;
        auto sub_column_ptr = IColumn::mutate(std::move(column_struct.get_column_ptr(i)));
        Defer defer_sub_column {
                [&] { column_struct.get_column_ptr(i) = std::move(sub_column_ptr); }};
        bool column_has_null = false;
        RETURN_IF_ERROR(
                _sub_column_iterators[i]->next_batch(&num_read, sub_column_ptr, &column_has_null));
        DCHECK(num_read == *n);
    }

    if (is_column_nullable(*dst) && need_to_read_meta_columns()) {
        size_t num_read = *n;
        auto null_map_ptr = static_cast<ColumnNullable&>(*dst).get_null_map_column_ptr();
        // in not-null to null linked-schemachange mode,
        // actually we do not change dat data include meta in footer,
        // so may dst from changed meta which is nullable but old data is not nullable,
        // if so, we should set null_map to all null by default
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
    }

    return Status::OK();
}

Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
    if (!need_to_read()) {
        DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    if (read_null_map_only()) {
        // In NULL_MAP_ONLY mode, only seek the null iterator; skip all sub-column iterators
        if (_struct_reader->is_nullable() && _null_iterator) {
            RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
        }
        return Status::OK();
    }

    for (auto& column_iterator : _sub_column_iterators) {
        RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord));
    }

    if (_struct_reader->is_nullable() && need_to_read_meta_columns()) {
        RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
    }
    return Status::OK();
}

Status StructFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& params) {
    for (auto& column_iterator : _sub_column_iterators) {
        RETURN_IF_ERROR(column_iterator->init_prefetcher(params));
    }
    if (_struct_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init_prefetcher(params));
    }
    return Status::OK();
}

void StructFileColumnIterator::collect_prefetchers(
        std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>& prefetchers,
        PrefetcherInitMethod init_method) {
    if (!need_to_read()) {
        return;
    }
    if (_struct_reader->is_nullable()) {
        _null_iterator->collect_prefetchers(prefetchers, init_method);
    }
    if (read_null_map_only()) {
        return;
    }
    for (auto& column_iterator : _sub_column_iterators) {
        if (column_iterator->need_to_read()) {
            column_iterator->collect_prefetchers(prefetchers, init_method);
        }
    }
}

Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
                                                MutableColumnPtr& dst) {
    if (!need_to_read()) {
        DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, count);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (count == 0) {
        return Status::OK();
    }

    size_t this_run = 1;
    auto start_idx = rowids[0];
    auto last_idx = rowids[0];
    for (size_t i = 1; i < count; ++i) {
        if (last_idx == rowids[i] - 1) {
            last_idx = rowids[i];
            this_run++;
            continue;
        }
        RETURN_IF_ERROR(seek_to_ordinal(start_idx));
        size_t num_read = this_run;
        RETURN_IF_ERROR(next_batch(&num_read, dst));
        DCHECK_EQ(num_read, this_run);

        start_idx = rowids[i];
        last_idx = rowids[i];
        this_run = 1;
    }

    RETURN_IF_ERROR(seek_to_ordinal(start_idx));
    size_t num_read = this_run;
    RETURN_IF_ERROR(next_batch(&num_read, dst));
    DCHECK_EQ(num_read, this_run);
    return Status::OK();
}

void StructFileColumnIterator::set_lazy_output_requirement() {
    set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    for (auto& sub_iterator : _sub_column_iterators) {
        sub_iterator->set_lazy_output_requirement();
    }
}

void StructFileColumnIterator::remove_pruned_sub_iterators() {
    for (auto it = _sub_column_iterators.begin(); it != _sub_column_iterators.end();) {
        auto& sub_iterator = *it;
        if (sub_iterator->read_requirement() == ReadRequirement::SKIP) {
            DLOG(INFO) << "Struct column iterator remove pruned sub-column "
                       << sub_iterator->column_name();
            it = _sub_column_iterators.erase(it);
        } else {
            sub_iterator->remove_pruned_sub_iterators();
            ++it;
        }
    }
}

Status StructFileColumnIterator::set_access_paths(
        const TColumnAccessPaths& all_access_paths,
        const TColumnAccessPaths& predicate_access_paths) {
    if (all_access_paths.empty() && predicate_access_paths.empty()) {
        return Status::OK();
    }

    auto plan = DORIS_TRY(_prepare_nested_access_paths(
            all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP,
            [this](ReadRequirement requirement) {
                for (auto& sub_iterator : _sub_column_iterators) {
                    sub_iterator->set_read_requirement(requirement);
                }
            }));

    if (plan.skip_data_descendants) {
        DLOG(INFO) << "Struct column iterator set column " << _column_name
                   << " to NULL_MAP_ONLY meta read mode, all sub-columns set to SKIP";
        return Status::OK();
    }

    const bool reads_all_sub_columns = plan.all.reads_current_data;
    const bool include_all_paths = !reads_all_sub_columns;
    for (auto& sub_iterator : _sub_column_iterators) {
        const auto name = sub_iterator->column_name();
        auto paths = DORIS_TRY(DescendantAccessPathRouter::select_struct_paths_for_child(
                plan.all.descendant_paths, plan.predicate.descendant_paths, name,
                include_all_paths));

        // Predicate paths must still reach the child even when no non-predicate path selects it.
        const bool need_to_read = reads_all_sub_columns || !paths.empty();
        if (!need_to_read) {
            set_read_requirement_self(ReadRequirement::SKIP);
            sub_iterator->set_read_requirement(ReadRequirement::SKIP);
            DLOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP";
            continue;
        }

        RETURN_IF_ERROR(sub_iterator->set_access_paths(paths.all_paths, paths.predicate_paths));
        // Set LAZY_OUTPUT after routing child predicate paths. If the child was needed only for
        // predicate evaluation, set_access_paths() has already promoted it to
        // PREDICATE and this monotonic update will not downgrade it. Otherwise, this
        // marks the child as a lazy materialization target.
        set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
        sub_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    }
    return Status::OK();
}

void StructFileColumnIterator::set_read_phase(ReadPhase mode) {
    ColumnIterator::set_read_phase(mode);
    for (auto& sub_iterator : _sub_column_iterators) {
        sub_iterator->set_read_phase(mode);
    }
}

void StructFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) {
    _recovery_from_place_holder_column(dst);
    auto& column_struct = assert_cast<ColumnStruct&, TypeCheckOnRelease::DISABLE>(
            dst->is_nullable() ? static_cast<ColumnNullable&>(*dst).get_nested_column() : *dst);

    for (size_t i = 0; i < _sub_column_iterators.size(); ++i) {
        auto& sub_column = column_struct.get_column_ptr(i);
        MutableColumnPtr mutable_sub_column = IColumn::mutate(std::move(sub_column));
        _sub_column_iterators[i]->finalize_lazy_phase(mutable_sub_column);
        sub_column = std::move(mutable_sub_column);
    }
}

void StructFileColumnIterator::set_read_requirement(ReadRequirement requirement) {
    set_read_requirement_self(requirement);
    for (const auto& sub_column_iterator : _sub_column_iterators) {
        sub_column_iterator->set_read_requirement(requirement);
    }
}

bool StructFileColumnIterator::has_lazy_read_target() const {
    if (_read_requirement == ReadRequirement::LAZY_OUTPUT) {
        return true;
    }
    return std::any_of(_sub_column_iterators.begin(), _sub_column_iterators.end(),
                       [](const auto& sub_column_iterator) {
                           return sub_column_iterator->has_lazy_read_target();
                       });
}

////////////////////////////////////////////////////////////////////////////////
Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) {
    RETURN_IF_ERROR(_offset_iterator->init(opts));
    // allocate peek tmp column once
    _peek_tmp_col = ColumnOffset64::create();
    return Status::OK();
}

Status OffsetFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    RETURN_IF_ERROR(_offset_iterator->next_batch(n, dst, has_null));
    return Status::OK();
}

Status OffsetFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
    if (_offset_iterator->get_current_page()->has_remaining()) {
        PageDecoder* offset_page_decoder = _offset_iterator->get_current_page()->data_decoder.get();
        size_t n = 1;
        _peek_tmp_col->clear();
        RETURN_IF_ERROR(offset_page_decoder->peek_next_batch(&n, _peek_tmp_col)); // not null
        DCHECK(_peek_tmp_col->size() == 1);
        *offset =
                assert_cast<const ColumnOffset64*, TypeCheckOnRelease::DISABLE>(_peek_tmp_col.get())
                        ->get_element(0);
    } else {
        *offset = _offset_iterator->get_current_page()->next_array_item_ordinal;
    }
    return Status::OK();
}

Status OffsetFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& params) {
    return _offset_iterator->init_prefetcher(params);
}

void OffsetFileColumnIterator::collect_prefetchers(
        std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>& prefetchers,
        PrefetcherInitMethod init_method) {
    _offset_iterator->collect_prefetchers(prefetchers, init_method);
}

/**
 *  first_storage_offset read from page should smaller than next_storage_offset which here call _peek_one_offset from page,
    and first_column_offset is keep in memory data which is different dimension with (first_storage_offset and next_storage_offset)
     eg. step1. read page: first_storage_offset = 16382
         step2. read page below with _peek_one_offset(&last_offset): last_offset = 16387
         step3. first_offset = 126 which is calculate in column offsets
         for loop column offsets element in size
            we can calculate from first_storage_offset to next_storage_offset one by one to fill with offsets_data in memory column offsets
 * @param start
 * @param column_offsets
 * @return
 */
Status OffsetFileColumnIterator::_calculate_offsets(ssize_t start,
                                                    ColumnArray::ColumnOffsets& column_offsets) {
    ordinal_t next_storage_offset = 0;
    RETURN_IF_ERROR(_peek_one_offset(&next_storage_offset));

    // calculate real offsets
    auto& offsets_data = column_offsets.get_data();
    ordinal_t first_column_offset = offsets_data[start - 1]; // -1 is valid
    ordinal_t first_storage_offset = offsets_data[start];
    DCHECK(next_storage_offset >= first_storage_offset);
    for (ssize_t i = start; i < offsets_data.size() - 1; ++i) {
        offsets_data[i] = first_column_offset + (offsets_data[i + 1] - first_storage_offset);
    }
    // last offset
    offsets_data[offsets_data.size() - 1] =
            first_column_offset + (next_storage_offset - first_storage_offset);
    return Status::OK();
}

////////////////////////////////////////////////////////////////////////////////
ArrayFileColumnIterator::ArrayFileColumnIterator(std::shared_ptr<ColumnReader> reader,
                                                 OffsetFileColumnIteratorUPtr offset_reader,
                                                 ColumnIteratorUPtr item_iterator,
                                                 ColumnIteratorUPtr null_iterator)
        : _array_reader(reader),
          _offset_iterator(std::move(offset_reader)),
          _item_iterator(std::move(item_iterator)) {
    if (_array_reader->is_nullable()) {
        _null_iterator = std::move(null_iterator);
    }
}

Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) {
    if (_read_requirement == ReadRequirement::SKIP) {
        DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    RETURN_IF_ERROR(_offset_iterator->init(opts));
    RETURN_IF_ERROR(_item_iterator->init(opts));
    if (_array_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init(opts));
    }
    return Status::OK();
}

Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) {
    if (read_offset_only()) {
        // In OFFSET_ONLY mode, item iterator is SKIP, no need to seek it
        return Status::OK();
    }
    // using offsets info
    ordinal_t offset = 0;
    RETURN_IF_ERROR(_offset_iterator->_peek_one_offset(&offset));
    RETURN_IF_ERROR(_item_iterator->seek_to_ordinal(offset));
    return Status::OK();
}

Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
    if (!need_to_read()) {
        DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    if (read_null_map_only()) {
        // In NULL_MAP_ONLY mode, only seek the null iterator; skip offset and item iterators
        if (_array_reader->is_nullable() && _null_iterator) {
            RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
        }
        return Status::OK();
    }

    RETURN_IF_ERROR(_offset_iterator->seek_to_ordinal(ord));
    if (_array_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
    }
    return _seek_by_offsets(ord);
}

Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    if (!need_to_read()) {
        DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading, read phase"
                   << static_cast<int>(_read_phase)
                   << ", read requirement: " << static_cast<int>(_read_requirement);
        _convert_to_place_holder_column(dst, *n);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        // NULL_MAP_ONLY mode: read null map, fill nested ColumnArray with empty defaults
        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        auto null_map_ptr = nullable_col.get_null_map_column_ptr();
        size_t num_read = *n;
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            // schema-change: column became nullable but old segment has no null data
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
        // fill nested ColumnArray with empty (zero-length) arrays
        auto& column_array = assert_cast<ColumnArray&, TypeCheckOnRelease::DISABLE>(
                nullable_col.get_nested_column());
        column_array.insert_many_defaults(num_read);
        *has_null = true;
        return Status::OK();
    }

    auto& column_array = assert_cast<ColumnArray&, TypeCheckOnRelease::DISABLE>(
            is_column_nullable(*dst) ? static_cast<ColumnNullable&>(*dst).get_nested_column()
                                     : *dst);

    bool offsets_has_null = false;
    const bool read_meta_columns = need_to_read_meta_columns();
    MutableColumnPtr column_offsets_ptr;
    if (read_meta_columns) {
        column_offsets_ptr = IColumn::mutate(std::move(column_array.get_offsets_ptr()));
    } else {
        const auto base_offset =
                column_array.get_offsets().empty() ? 0 : column_array.get_offsets().back();
        column_offsets_ptr = ColumnArray::ColumnOffsets::create();
        assert_cast<ColumnOffset64&, TypeCheckOnRelease::DISABLE>(*column_offsets_ptr)
                .insert_value(base_offset);
    }
    Defer defer_offsets {[&] {
        if (read_meta_columns) {
            auto typed_column_offsets_ptr = ColumnArray::ColumnOffsets::cast_to_column_mutptr(
                    assert_cast<ColumnArray::ColumnOffsets*, TypeCheckOnRelease::DISABLE>(
                            column_offsets_ptr.get()));
            column_offsets_ptr = nullptr;
            column_array.get_offsets_ptr() = std::move(typed_column_offsets_ptr);
        }
    }};
    ssize_t start = column_offsets_ptr->size();
    RETURN_IF_ERROR(_offset_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null));
    if (*n == 0) {
        return Status::OK();
    }
    auto& column_offsets = static_cast<ColumnArray::ColumnOffsets&>(*column_offsets_ptr);
    RETURN_IF_ERROR(_offset_iterator->_calculate_offsets(start, column_offsets));
    size_t num_items =
            column_offsets.get_data().back() - column_offsets.get_data()[start - 1]; // -1 is valid
    if (num_items > 0) {
        auto column_items_ptr = IColumn::mutate(std::move(column_array.get_data_ptr()));
        Defer defer_items {[&] { column_array.get_data_ptr() = std::move(column_items_ptr); }};
        if (read_offset_only()) {
            // OFFSET_ONLY mode: skip reading actual item data, fill with defaults
            column_items_ptr->insert_many_defaults(num_items);
        } else {
            size_t num_read = num_items;
            bool items_has_null = false;
            RETURN_IF_ERROR(
                    _item_iterator->next_batch(&num_read, column_items_ptr, &items_has_null));
            DCHECK(num_read == num_items);
        }
    }

    if (is_column_nullable(*dst) && read_meta_columns) {
        auto null_map_ptr = static_cast<ColumnNullable&>(*dst).get_null_map_column_ptr();
        size_t num_read = *n;
        // in not-null to null linked-schemachange mode,
        // actually we do not change dat data include meta in footer,
        // so may dst from changed meta which is nullable but old data is not nullable,
        // if so, we should set null_map to all null by default
        if (_null_iterator) {
            bool null_signs_has_null = false;
            MutableColumnPtr null_map_column = std::move(null_map_ptr);
            RETURN_IF_ERROR(
                    _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null));
        } else {
            null_map_ptr->insert_many_vals(0, num_read);
        }
        DCHECK(num_read == *n);
    }

    return Status::OK();
}

Status ArrayFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& params) {
    RETURN_IF_ERROR(_offset_iterator->init_prefetcher(params));
    RETURN_IF_ERROR(_item_iterator->init_prefetcher(params));
    if (_array_reader->is_nullable()) {
        RETURN_IF_ERROR(_null_iterator->init_prefetcher(params));
    }
    return Status::OK();
}

void ArrayFileColumnIterator::collect_prefetchers(
        std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>& prefetchers,
        PrefetcherInitMethod init_method) {
    if (!need_to_read()) {
        return;
    }
    if (!read_null_map_only()) {
        _offset_iterator->collect_prefetchers(prefetchers, init_method);
    }
    if (_array_reader->is_nullable()) {
        _null_iterator->collect_prefetchers(prefetchers, init_method);
    }
    if (read_offset_only() || read_null_map_only()) {
        return;
    }
    // the actual data pages to read of item column depends on the read result of offset column,
    // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here.
    if (_item_iterator->need_to_read()) {
        _item_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS);
    }
}

Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
                                               MutableColumnPtr& dst) {
    if (!need_to_read()) {
        DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, count);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    for (size_t i = 0; i < count; ++i) {
        // TODO(cambyszju): now read array one by one, need optimize later
        RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
        size_t num_read = 1;
        RETURN_IF_ERROR(next_batch(&num_read, dst));
    }
    return Status::OK();
}

void ArrayFileColumnIterator::set_lazy_output_requirement() {
    set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    _item_iterator->set_lazy_output_requirement();
}

void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
    _item_iterator->remove_pruned_sub_iterators();
}

void ArrayFileColumnIterator::set_read_phase(ReadPhase mode) {
    ColumnIterator::set_read_phase(mode);
    _item_iterator->set_read_phase(mode);
}

void ArrayFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) {
    _recovery_from_place_holder_column(dst);
    auto& column_array = assert_cast<ColumnArray&, TypeCheckOnRelease::DISABLE>(
            dst->is_nullable() ? static_cast<ColumnNullable&>(*dst).get_nested_column() : *dst);
    auto item_column_ptr = IColumn::mutate(std::move(column_array.get_data_ptr()));
    _item_iterator->finalize_lazy_phase(item_column_ptr);
    column_array.get_data_ptr() = std::move(item_column_ptr);
}

void ArrayFileColumnIterator::set_read_requirement(ReadRequirement requirement) {
    set_read_requirement_self(requirement);
    _item_iterator->set_read_requirement(requirement);
}

bool ArrayFileColumnIterator::has_lazy_read_target() const {
    return _read_requirement == ReadRequirement::LAZY_OUTPUT ||
           _item_iterator->has_lazy_read_target();
}

Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
                                                 const TColumnAccessPaths& predicate_access_paths) {
    if (all_access_paths.empty() && predicate_access_paths.empty()) {
        return Status::OK();
    }

    auto plan = DORIS_TRY(_prepare_nested_access_paths(
            all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP_AND_OFFSET,
            [this](ReadRequirement requirement) {
                _item_iterator->set_read_requirement(requirement);
            }));

    if (plan.skip_data_descendants) {
        return Status::OK();
    }

    const bool has_all_descendant = plan.all.has_descendant_paths();
    const bool has_predicate_descendant = plan.predicate.has_descendant_paths();
    auto child_paths = DORIS_TRY(DescendantAccessPathRouter::route_array_paths_to_item(
            std::move(plan.all.descendant_paths), std::move(plan.predicate.descendant_paths),
            _item_iterator->column_name()));

    if (has_all_descendant || has_predicate_descendant) {
        RETURN_IF_ERROR(_item_iterator->set_access_paths(child_paths.all_paths,
                                                         child_paths.predicate_paths));
        // Predicate-only item paths stay PREDICATE because this update runs after
        // child set_access_paths() and read requirements are monotonic. Non-predicate item paths are
        // marked as lazy materialization targets.
        _item_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT);
    }
    return Status::OK();
}

////////////////////////////////////////////////////////////////////////////////
// StringFileColumnIterator implementation
////////////////////////////////////////////////////////////////////////////////

StringFileColumnIterator::StringFileColumnIterator(std::shared_ptr<ColumnReader> reader)
        : FileColumnIterator(std::move(reader)) {}

Status StringFileColumnIterator::init(const ColumnIteratorOptions& opts) {
    if (read_offset_only()) {
        // Propagate only_read_offsets to the FileColumnIterator's options
        auto modified_opts = opts;
        modified_opts.only_read_offsets = true;
        return FileColumnIterator::init(modified_opts);
    }
    return FileColumnIterator::init(opts);
}

Status StringFileColumnIterator::set_access_paths(
        const TColumnAccessPaths& all_access_paths,
        const TColumnAccessPaths& predicate_access_paths) {
    RETURN_IF_ERROR(FileColumnIterator::set_access_paths(all_access_paths, predicate_access_paths));
    // OFFSET_ONLY mode is fundamentally incompatible with CHAR columns:
    // CHAR is stored padded to its declared length (see
    // OlapColumnDataConvertorChar::clone_and_padding), so the per-row length
    // recorded in dict word info / page headers is always the padded length
    // (e.g. 25 for CHAR(25)) — never the logical length expected by length().
    // Recovering the logical length requires scanning the chars buffer with
    // strnlen() (shrink_padding_chars), which OFFSET_ONLY by definition skips.
    // There is no partial-benefit path: any optimization that still produces
    // the correct length() result must read the chars buffer in full.
    //
    // FE (NestedColumnPruning) already filters CHAR slots out of the
    // OFFSET-only access plan, so reaching this branch means an FE/BE
    // contract violation. Fail loudly instead of silently falling back.
    if (read_offset_only() && get_reader() != nullptr &&
        get_reader()->get_meta_type() == FieldType::OLAP_FIELD_TYPE_CHAR) {
        return Status::InternalError(
                "OFFSET_ONLY access path is not supported on CHAR column '{}': CHAR is stored "
                "padded so the per-row length information available without reading the chars "
                "buffer is always the padded length, not the logical length. The FE planner "
                "must not emit an OFFSET access path for CHAR columns.",
                _column_name);
    }
    if (read_offset_only()) {
        DLOG(INFO) << "String column iterator set column " << _column_name
                   << " to OFFSET_ONLY meta read mode";
    } else if (read_null_map_only()) {
        DLOG(INFO) << "String column iterator set column " << _column_name
                   << " to NULL_MAP_ONLY meta read mode";
    }

    return Status::OK();
}

////////////////////////////////////////////////////////////////////////////////

Status FileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
                                            const TColumnAccessPaths& predicate_access_paths) {
    if (all_access_paths.empty() && predicate_access_paths.empty()) {
        return Status::OK();
    }

    const auto requirement_before_access_path = _read_requirement;
    if (!predicate_access_paths.empty()) {
        set_read_requirement(ReadRequirement::PREDICATE);
    }

    auto all_split = DORIS_TRY(_split_access_paths(all_access_paths));
    auto predicate_split = DORIS_TRY(_split_access_paths(predicate_access_paths));
    if (all_split.reads_current_data) {
        set_lazy_output_requirement();
    }
    if (predicate_split.reads_current_data) {
        set_read_requirement(ReadRequirement::PREDICATE);
    }
    return _check_and_set_meta_read_mode(requirement_before_access_path, all_split);
}

FileColumnIterator::FileColumnIterator(std::shared_ptr<ColumnReader> reader) : _reader(reader) {}

Status ColumnIterator::_check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path,
                                                     const AccessPathSplit& all_access_paths) {
    _meta_read_mode = MetaReadMode::DEFAULT;
    if (requirement_before_access_path != ReadRequirement::NORMAL &&
        requirement_before_access_path != ReadRequirement::SKIP) {
        // A stronger requirement means a parent/full-data path already required this iterator
        // to materialize data. In that case a later predicate NULL/OFFSET path is only
        // an additional predicate requirement and must not downgrade the read to
        // meta-only.
        return Status::OK();
    }

    if (all_access_paths.reads_current_data || all_access_paths.has_descendant_paths()) {
        return Status::OK();
    }

    _meta_read_mode = all_access_paths.current_meta_mode;
    return Status::OK();
}

Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
    if (_read_requirement == ReadRequirement::SKIP) {
        DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    _opts = opts;
    if (!_opts.use_page_cache) {
        _reader->disable_index_meta_cache();
    }
    RETURN_IF_ERROR(get_block_compression_codec(_reader->get_compression(), &_compress_codec));
    if (config::enable_low_cardinality_optimize &&
        opts.io_ctx.reader_type == ReaderType::READER_QUERY &&
        _reader->encoding_info()->encoding() == DICT_ENCODING) {
        auto dict_encoding_type = _reader->get_dict_encoding_type();
        // Only if the column is a predicate column, then we need check the all dict encoding flag
        // because we could rewrite the predciate to accelarate query speed. But if it is not a
        // predicate column, then it is useless. And it has a bad impact on cold read(first time read)
        // because it will load the column's ordinal index and zonemap index and maybe other indices.
        // it has bad impact on primary key query. For example, select * from table where pk = 1, and
        // the table has 2000 columns.
        if (dict_encoding_type == ColumnReader::UNKNOWN_DICT_ENCODING && opts.is_predicate_column) {
            RETURN_IF_ERROR(seek_to_ordinal(_reader->num_rows() - 1));
            _is_all_dict_encoding = _page.is_dict_encoding;
            _reader->set_dict_encoding_type(_is_all_dict_encoding
                                                    ? ColumnReader::ALL_DICT_ENCODING
                                                    : ColumnReader::PARTIAL_DICT_ENCODING);
        } else {
            _is_all_dict_encoding = dict_encoding_type == ColumnReader::ALL_DICT_ENCODING;
        }
    }
    return Status::OK();
}

FileColumnIterator::~FileColumnIterator() = default;

void FileColumnIterator::_trigger_prefetch_if_eligible(ordinal_t ord) {
    std::vector<BlockRange> ranges;
    if (_prefetcher->need_prefetch(cast_set<uint32_t>(ord), &ranges)) {
        for (const auto& range : ranges) {
            _cached_remote_file_reader->prefetch_range(range.offset, range.size, &_opts.io_ctx);
        }
    }
}

Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
    if (!need_to_read()) {
        DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
        return Status::OK();
    }

    LOG_IF(INFO, config::enable_segment_prefetch_verbose_log) << fmt::format(
            "[verbose] FileColumnIterator::seek_to_ordinal seek to ordinal {}, enable_prefetch={}",
            ord, _enable_prefetch);
    if (_enable_prefetch) {
        _trigger_prefetch_if_eligible(ord);
    }

    // if current page contains this row, we don't need to seek
    if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
        RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
        RETURN_IF_ERROR(_read_data_page(_page_iter));
    }
    RETURN_IF_ERROR(_seek_to_pos_in_page(&_page, ord - _page.first_ordinal));
    _current_ordinal = ord;
    return Status::OK();
}

Status FileColumnIterator::seek_to_page_start() {
    return seek_to_ordinal(_page.first_ordinal);
}

Status FileColumnIterator::_seek_to_pos_in_page(ParsedPage* page, ordinal_t offset_in_page) const {
    if (page->offset_in_page == offset_in_page) {
        // fast path, do nothing
        return Status::OK();
    }

    ordinal_t pos_in_data = offset_in_page;
    if (_page.has_null) {
        ordinal_t offset_in_data = 0;
        ordinal_t skips = offset_in_page;

        if (offset_in_page > page->offset_in_page) {
            // forward, reuse null bitmap
            skips = offset_in_page - page->offset_in_page;
            offset_in_data = page->data_decoder->current_index();
        } else {
            // rewind null bitmap, and
            page->null_decoder = RleDecoder<bool>((const uint8_t*)page->null_bitmap.data,
                                                  cast_set<int>(page->null_bitmap.size), 1);
        }

        auto skip_nulls = page->null_decoder.Skip(skips);
        pos_in_data = offset_in_data + skips - skip_nulls;
    }

    RETURN_IF_ERROR(page->data_decoder->seek_to_position_in_page(pos_in_data));
    page->offset_in_page = offset_in_page;
    return Status::OK();
}

Status FileColumnIterator::next_batch_of_zone_map(size_t* n, MutableColumnPtr& dst) {
    return _reader->next_batch_of_zone_map(n, dst);
}

Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    if (!need_to_read()) {
        DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, *n);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        DLOG(INFO) << "File column iterator column " << _column_name
                   << " in NULL_MAP_ONLY mode, reading only null map.";
        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        auto& null_map_data = nullable_col.get_null_map_data();

        size_t remaining = *n;
        *has_null = false;
        while (remaining > 0) {
            if (!_page.has_remaining()) {
                bool eos = false;
                RETURN_IF_ERROR(_load_next_page(&eos));
                if (eos) {
                    break;
                }
            }

            size_t nrows_in_page = std::min(remaining, _page.remaining());
            size_t nrows_to_read = nrows_in_page;
            if (_page.has_null) {
                while (nrows_to_read > 0) {
                    bool is_null = false;
                    size_t this_run = _page.null_decoder.GetNextRun(&is_null, nrows_to_read);
                    const size_t cur_size = null_map_data.size();
                    null_map_data.resize(cur_size + this_run);
                    memset(null_map_data.data() + cur_size, is_null ? 1 : 0, this_run);
                    if (is_null) {
                        *has_null = true;
                    }
                    nrows_to_read -= this_run;
                    _page.offset_in_page += this_run;
                    _current_ordinal += this_run;
                }
            } else {
                const size_t cur_size = null_map_data.size();
                null_map_data.resize(cur_size + nrows_to_read);
                memset(null_map_data.data() + cur_size, 0, nrows_to_read);
                _page.offset_in_page += nrows_to_read;
                _current_ordinal += nrows_to_read;
            }
            remaining -= nrows_in_page;
        }
        *n -= remaining;
        nullable_col.get_nested_column().insert_many_defaults(*n);
        return Status::OK();
    }

    size_t curr_size = dst->byte_size();
    dst->reserve(*n);
    size_t remaining = *n;
    *has_null = false;
    while (remaining > 0) {
        if (!_page.has_remaining()) {
            bool eos = false;
            RETURN_IF_ERROR(_load_next_page(&eos));
            if (eos) {
                break;
            }
        }

        // number of rows to be read from this page
        size_t nrows_in_page = std::min(remaining, _page.remaining());
        size_t nrows_to_read = nrows_in_page;
        if (_page.has_null) {
            while (nrows_to_read > 0) {
                bool is_null = false;
                size_t this_run = _page.null_decoder.GetNextRun(&is_null, nrows_to_read);
                // we use num_rows only for CHECK
                size_t num_rows = this_run;
                if (!is_null) {
                    RETURN_IF_ERROR(_page.data_decoder->next_batch(&num_rows, dst));
                    DCHECK_EQ(this_run, num_rows);
                } else {
                    *has_null = true;
                    auto* null_col = check_and_get_column<ColumnNullable>(dst.get());
                    if (null_col != nullptr) {
                        null_col->insert_many_defaults(this_run);
                    } else {
                        return Status::InternalError("unexpected column type in column reader");
                    }
                }

                nrows_to_read -= this_run;
                _page.offset_in_page += this_run;
                _current_ordinal += this_run;
            }
        } else {
            RETURN_IF_ERROR(_page.data_decoder->next_batch(&nrows_to_read, dst));
            DCHECK_EQ(nrows_to_read, nrows_in_page);

            _page.offset_in_page += nrows_to_read;
            _current_ordinal += nrows_to_read;
        }
        remaining -= nrows_in_page;
    }
    *n -= remaining;
    _opts.stats->bytes_read += (dst->byte_size() - curr_size) + BitmapSize(*n);

#ifdef BE_TEST
    _reader->check_data_by_zone_map_for_test(dst);
#endif
    return Status::OK();
}

Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
                                          MutableColumnPtr& dst) {
    if (!need_to_read()) {
        DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
        _convert_to_place_holder_column(dst, count);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);

    if (read_null_map_only()) {
        DLOG(INFO) << "File column iterator column " << _column_name
                   << " in NULL_MAP_ONLY mode, reading only null map by rowids.";

        DORIS_CHECK(is_column_nullable(*dst));
        auto& nullable_col = assert_cast<ColumnNullable&>(*dst);
        auto& null_map_data = nullable_col.get_null_map_data();
        const size_t base_size = null_map_data.size();
        null_map_data.resize(base_size + count);

        size_t remaining = count;
        size_t total_read_count = 0;
        size_t nrows_to_read = 0;
        while (remaining > 0) {
            RETURN_IF_ERROR(seek_to_ordinal(rowids[total_read_count]));

            nrows_to_read = std::min(remaining, _page.remaining());

            if (_page.has_null) {
                size_t already_read = 0;
                while ((nrows_to_read - already_read) > 0) {
                    bool is_null = false;
                    size_t this_run = std::min(nrows_to_read - already_read, _page.remaining());
                    if (UNLIKELY(this_run == 0)) {
                        break;
                    }
                    this_run = _page.null_decoder.GetNextRun(&is_null, this_run);

                    size_t offset = total_read_count + already_read;
                    size_t this_read_count = 0;
                    rowid_t current_ordinal_in_page =
                            cast_set<uint32_t>(_page.offset_in_page + _page.first_ordinal);
                    for (size_t i = 0; i < this_run; ++i) {
                        if (rowids[offset + i] - current_ordinal_in_page >= this_run) {
                            break;
                        }
                        this_read_count++;
                    }

                    if (this_read_count > 0) {
                        memset(null_map_data.data() + base_size + offset, is_null ? 1 : 0,
                               this_read_count);
                    }

                    already_read += this_read_count;
                    _page.offset_in_page += this_run;
                }

                nrows_to_read = already_read;
                total_read_count += nrows_to_read;
                remaining -= nrows_to_read;
            } else {
                rowid_t current_ordinal_in_page =
                        cast_set<uint32_t>(_page.offset_in_page + _page.first_ordinal);
                size_t rows_in_current_page = 0;
                for (size_t i = 0; i < nrows_to_read; ++i) {
                    if (rowids[total_read_count + i] - current_ordinal_in_page >= nrows_to_read) {
                        break;
                    }
                    ++rows_in_current_page;
                }
                DCHECK_GT(rows_in_current_page, 0);
                memset(null_map_data.data() + base_size + total_read_count, 0,
                       rows_in_current_page);
                _page.offset_in_page += rows_in_current_page;
                total_read_count += rows_in_current_page;
                remaining -= rows_in_current_page;
            }
        }

        null_map_data.resize(base_size + total_read_count);
        nullable_col.get_nested_column().insert_many_defaults(total_read_count);
        return Status::OK();
    }

    size_t remaining = count;
    size_t total_read_count = 0;
    size_t nrows_to_read = 0;
    while (remaining > 0) {
        RETURN_IF_ERROR(seek_to_ordinal(rowids[total_read_count]));

        // number of rows to be read from this page
        nrows_to_read = std::min(remaining, _page.remaining());

        if (_page.has_null) {
            size_t already_read = 0;
            while ((nrows_to_read - already_read) > 0) {
                bool is_null = false;
                size_t this_run = std::min(nrows_to_read - already_read, _page.remaining());
                if (UNLIKELY(this_run == 0)) {
                    break;
                }
                this_run = _page.null_decoder.GetNextRun(&is_null, this_run);
                size_t offset = total_read_count + already_read;
                size_t this_read_count = 0;
                rowid_t current_ordinal_in_page =
                        cast_set<uint32_t>(_page.offset_in_page + _page.first_ordinal);
                for (size_t i = 0; i < this_run; ++i) {
                    if (rowids[offset + i] - current_ordinal_in_page >= this_run) {
                        break;
                    }
                    this_read_count++;
                }

                auto origin_index = _page.data_decoder->current_index();
                if (this_read_count > 0) {
                    if (is_null) {
                        auto* null_col = check_and_get_column<ColumnNullable>(dst.get());
                        if (UNLIKELY(null_col == nullptr)) {
                            return Status::InternalError("unexpected column type in column reader");
                        }

                        null_col->insert_many_defaults(this_read_count);
                    } else {
                        size_t read_count = this_read_count;

                        // ordinal in nullable columns' data buffer maybe be not continuously(the data doesn't contain null value),
                        // so we need use `page_start_off_in_decoder` to calculate the actual offset in `data_decoder`
                        size_t page_start_off_in_decoder =
                                _page.first_ordinal + _page.offset_in_page - origin_index;
                        RETURN_IF_ERROR(_page.data_decoder->read_by_rowids(
                                &rowids[offset], page_start_off_in_decoder, &read_count, dst));
                        DCHECK_EQ(read_count, this_read_count);
                    }
                }

                if (!is_null) {
                    RETURN_IF_ERROR(
                            _page.data_decoder->seek_to_position_in_page(origin_index + this_run));
                }

                already_read += this_read_count;
                _page.offset_in_page += this_run;
                DCHECK(_page.offset_in_page <= _page.num_rows);
            }

            nrows_to_read = already_read;
            total_read_count += nrows_to_read;
            remaining -= nrows_to_read;
        } else {
            RETURN_IF_ERROR(_page.data_decoder->read_by_rowids(
                    &rowids[total_read_count], _page.first_ordinal, &nrows_to_read, dst));
            total_read_count += nrows_to_read;
            remaining -= nrows_to_read;
        }
    }
    return Status::OK();
}

Status FileColumnIterator::_load_next_page(bool* eos) {
    _page_iter.next();
    if (!_page_iter.valid()) {
        *eos = true;
        return Status::OK();
    }

    RETURN_IF_ERROR(_read_data_page(_page_iter));
    RETURN_IF_ERROR(_seek_to_pos_in_page(&_page, 0));
    *eos = false;
    return Status::OK();
}

Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) {
    PageHandle handle;
    Slice page_body;
    PageFooterPB footer;
    _opts.type = DATA_PAGE;
    PageDecoderOptions decoder_opts;
    decoder_opts.only_read_offsets = _opts.only_read_offsets;
    RETURN_IF_ERROR(
            _reader->read_page(_opts, iter.page(), &handle, &page_body, &footer, _compress_codec));
    // parse data page
    auto st = ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(),
                                 _reader->encoding_info(), iter.page(), iter.page_index(), &_page,
                                 decoder_opts);
    if (!st.ok()) {
        LOG(WARNING) << "failed to create ParsedPage, file=" << _opts.file_reader->path().native()
                     << ", page_offset=" << iter.page().offset << ", page_size=" << iter.page().size
                     << ", page_index=" << iter.page_index() << ", error=" << st;
        return st;
    }

    // dictionary page is read when the first data page that uses it is read,
    // this is to optimize the memory usage: when there is no query on one column, we could
    // release the memory of dictionary page.
    // note that concurrent iterators for the same column won't repeatedly read dictionary page
    // because of page cache.
    if (_reader->encoding_info()->encoding() == DICT_ENCODING) {
        auto dict_page_decoder = reinterpret_cast<BinaryDictPageDecoder*>(_page.data_decoder.get());
        if (dict_page_decoder->is_dict_encoding()) {
            if (_dict_decoder == nullptr) {
                RETURN_IF_ERROR(_read_dict_data());
                CHECK_NOTNULL(_dict_decoder);
            }

            dict_page_decoder->set_dict_decoder(cast_set<uint32_t>(_dict_decoder->count()),
                                                _dict_word_info.get());
        }
    }
    return Status::OK();
}

Status FileColumnIterator::_read_dict_data() {
    CHECK_EQ(_reader->encoding_info()->encoding(), DICT_ENCODING);
    // read dictionary page
    Slice dict_data;
    PageFooterPB dict_footer;
    _opts.type = INDEX_PAGE;

    RETURN_IF_ERROR(_reader->read_page(_opts, _reader->get_dict_page_pointer(), &_dict_page_handle,
                                       &dict_data, &dict_footer, _compress_codec));
    const EncodingInfo* encoding_info;
    // The dict pool stores strings of the outer column's type. Using the
    // outer type (CHAR vs VARCHAR/STRING) lets the EncodingInfo pick a
    // CHAR-strip pre-decoder so the cached dict page is already unpadded.
    RETURN_IF_ERROR(EncodingInfo::get(_reader->get_meta_type(),
                                      dict_footer.dict_page_footer().encoding(), &encoding_info));
    RETURN_IF_ERROR(encoding_info->create_page_decoder(dict_data, {}, _dict_decoder));
    RETURN_IF_ERROR(_dict_decoder->init());

    _dict_word_info.reset(new StringRef[_dict_decoder->count()]);
    RETURN_IF_ERROR(_dict_decoder->get_dict_word_info(_dict_word_info.get()));
    return Status::OK();
}

Status FileColumnIterator::get_row_ranges_by_zone_map(
        const AndBlockColumnPredicate* col_predicates,
        const std::vector<std::shared_ptr<const ColumnPredicate>>* delete_predicates,
        RowRanges* row_ranges) {
    if (_reader->has_zone_map()) {
        RETURN_IF_ERROR(_reader->get_row_ranges_by_zone_map(col_predicates, delete_predicates,
                                                            row_ranges, _opts));
    }
    return Status::OK();
}

Status FileColumnIterator::get_row_ranges_by_bloom_filter(
        const AndBlockColumnPredicate* col_predicates, RowRanges* row_ranges) {
    if ((col_predicates->can_do_bloom_filter(false) && _reader->has_bloom_filter_index(false)) ||
        (col_predicates->can_do_bloom_filter(true) && _reader->has_bloom_filter_index(true))) {
        RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates, row_ranges, _opts));
    }
    return Status::OK();
}

Status FileColumnIterator::get_row_ranges_by_dict(const AndBlockColumnPredicate* col_predicates,
                                                  RowRanges* row_ranges) {
    if (!_is_all_dict_encoding) {
        return Status::OK();
    }

    if (!_dict_decoder) {
        RETURN_IF_ERROR(_read_dict_data());
        CHECK_NOTNULL(_dict_decoder);
    }

    if (!col_predicates->evaluate_and(_dict_word_info.get(), _dict_decoder->count())) {
        row_ranges->clear();
    }
    return Status::OK();
}

Status FileColumnIterator::init_prefetcher(const SegmentPrefetchParams& params) {
    if (_cached_remote_file_reader =
                std::dynamic_pointer_cast<io::CachedRemoteFileReader>(_reader->_file_reader);
        !_cached_remote_file_reader) {
        return Status::OK();
    }
    _enable_prefetch = true;
    _prefetcher = std::make_unique<SegmentPrefetcher>(params.config);
    RETURN_IF_ERROR(_prefetcher->init(_reader, params.read_options));
    return Status::OK();
}

void FileColumnIterator::collect_prefetchers(
        std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>& prefetchers,
        PrefetcherInitMethod init_method) {
    if (_prefetcher) {
        prefetchers[init_method].emplace_back(_prefetcher.get());
    }
}

Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) {
    _opts = opts;
    // be consistent with segment v1
    // if _has_default_value, we should create default column iterator for this column, and
    // "NULL" is a special default value which means the default value is null.
    if (_has_default_value) {
        if (_default_value == "NULL") {
            _default_value_field = Field::create_field<TYPE_NULL>(Null {});
        } else {
            if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
                if (_default_value != "[]") {
                    return Status::NotSupported("Array default {} is unsupported", _default_value);
                } else {
                    _default_value_field = Field::create_field<TYPE_ARRAY>(Array {});
                    return Status::OK();
                }
            } else if (_type == FieldType::OLAP_FIELD_TYPE_STRUCT) {
                return Status::NotSupported("STRUCT default type is unsupported");
            } else if (_type == FieldType::OLAP_FIELD_TYPE_MAP) {
                return Status::NotSupported("MAP default type is unsupported");
            }
            const auto t = _type;
            const auto serde = DataTypeFactory::instance()
                                       .create_data_type(t, _precision, _scale, _len)
                                       ->get_serde();
            RETURN_IF_ERROR(serde->from_fe_string(_default_value, _default_value_field));
        }
    } else if (_is_nullable) {
        _default_value_field = Field::create_field<TYPE_NULL>(Null {});
    } else {
        return Status::InternalError(
                "invalid default value column for no default value and not nullable");
    }
    return Status::OK();
}

Status DefaultValueColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    if (!need_to_read()) {
        _convert_to_place_holder_column(dst, *n);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);
    *has_null = _default_value_field.is_null();
    _insert_many_default(dst, *n);
    return Status::OK();
}

Status DefaultValueColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
                                                  MutableColumnPtr& dst) {
    if (!need_to_read()) {
        _convert_to_place_holder_column(dst, count);
        return Status::OK();
    }

    _recovery_from_place_holder_column(dst);
    _insert_many_default(dst, count);
    return Status::OK();
}

void DefaultValueColumnIterator::_insert_many_default(MutableColumnPtr& dst, size_t n) {
    if (_default_value_field.is_null()) {
        dst->insert_many_defaults(n);
    } else {
        dst = dst->convert_to_predicate_column_if_dictionary();
        dst->insert_duplicate_fields(_default_value_field, n);
    }
}

Status RowIdColumnIteratorV2::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) {
    auto* string_column = assert_cast<ColumnString*, TypeCheckOnRelease::DISABLE>(dst.get());

    for (uint32_t i = 0; i < *n; ++i) {
        uint32_t row_id = _current_rowid + i;
        GlobalRowLoacationV2 location(_version, _backend_id, _file_id, row_id);
        string_column->insert_data(reinterpret_cast<const char*>(&location),
                                   sizeof(GlobalRowLoacationV2));
    }
    _current_rowid += *n;
    return Status::OK();
}

Status RowIdColumnIteratorV2::read_by_rowids(const rowid_t* rowids, const size_t count,
                                             MutableColumnPtr& dst) {
    auto* string_column = assert_cast<ColumnString*>(dst.get());

    for (size_t i = 0; i < count; ++i) {
        uint32_t row_id = rowids[i];
        GlobalRowLoacationV2 location(_version, _backend_id, _file_id, row_id);
        string_column->insert_data(reinterpret_cast<const char*>(&location),
                                   sizeof(GlobalRowLoacationV2));
    }
    return Status::OK();
}

} // namespace doris::segment_v2

Coverage Report

Created: 2026-07-20 18:23