// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//   http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.

#pragma once

#include <algorithm>

#include <cstddef>

#include <cstdint>

#include <functional>

#include <limits>

#include <map>

#include <memory>

#include <string>

#include <utility>

#include <vector>

#include "common/cast_set.h"

#include "common/status.h"

#include "core/data_type/data_type.h"

#include "core/field.h"

#include "exprs/vexpr_fwd.h"

#include "format_v2/column_data.h"

#include "gen_cpp/PlanNodes_types.h"

#include "io/file_factory.h"

#include "io/fs/file_reader_writer_fwd.h"

#include "io/io_common.h"

namespace doris {

class Block;

struct ConditionCacheContext;

namespace io {

struct IOContext;

} // namespace io

} // namespace doris

namespace doris::format {

class TableColumnMapper;

struct TableColumnMapperOptions;

enum class FileFormat {

    PARQUET,

    ORC,

    CSV,

    JSON,

    TEXT,

    JNI,

    ARROW,

    WAL,

    ES_HTTP,

};

struct FileScanRequest {

    virtual ~FileScanRequest() = default;

    std::string debug_string() const;

    // Columns that must be read before row-level filtering. They are materialized eagerly because

    // conjuncts/delete_conjuncts need them to decide the selected rows.

    std::vector<LocalColumnIndex> predicate_columns;

    // Columns read after row-level filtering. A complex root may intentionally also appear in

    // predicate_columns when its eager predicate subtree is smaller than its final output subtree.

    std::vector<LocalColumnIndex> non_predicate_columns;

    // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead

    // after all file-local predicates run, although the shared file block still needs row-shaped

    // placeholders until TableReader finalizes projected columns.

    std::vector<LocalColumnId> predicate_only_columns;

    // file-local column id -> file-local output block position.

    std::map<LocalColumnId, LocalIndex> local_positions;

    // Optional output position for a root that has independent eager-predicate and deferred-output

    // projections. local_positions continues to identify the position referenced by localized

    // predicate expressions.

    std::map<LocalColumnId, LocalIndex> non_predicate_positions;

    // Row-level filters converted to file-local expressions from table-level predicates.

    VExprContextSPtrs conjuncts;

    // Only this leading subset may participate in footer/page metadata pruning. The boundary is

    // inherited from table-conjunct order so an omitted slotless unsafe expression remains a fence.

    size_t metadata_pruning_safe_conjunct_count = std::numeric_limits<size_t>::max();

    // Constant split pruning may use only this table-filter prefix after mapping. A rejected

    // file-local rewrite is a materialization barrier even when a later filter is constant.

    size_t constant_pruning_safe_table_filter_count = std::numeric_limits<size_t>::max();

    // Delete predicates converted to file-local expressions. A TRUE result means that the row is

    // deleted, so readers must invert each result when building their keep filter.

    VExprContextSPtrs delete_conjuncts;

    // File-local ids retained only because Nereids keeps a minimum-width output tuple for an

    // explicit COUNT(*). These columns have no semantic value: for example, after pruning a scan

    // may retain an unsupported TIME_MILLIS leaf even though COUNT(*) only needs one row per

    // surviving input row. A reader may synthesize defaults instead of reading a marked column

    // while it remains non-predicate. If filters or equality deletes promote the same id to

    // predicate_columns, the value is semantically required and must still be validated and read.

    std::vector<LocalColumnId> count_star_placeholder_columns;

    // Absolute zero-based file row positions selected by a row-id fetch. A present but empty

    // vector means that no rows should be read; nullopt keeps the normal sequential scan path.

    // Readers require strictly increasing positions so they can seek forward without duplicating

    // output rows.

    std::optional<std::vector<int64_t>> row_ids = std::nullopt;

    // Table formats may assign semantics that legacy physical files do not encode. Each path here

    // identifies an unannotated Parquet group that the physical reader must validate and decode as

    // Variant. Keeping this explicit prevents generic Parquet scans from guessing based on names.

    std::vector<LocalColumnIndex> variant_schema_overrides;

    bool is_count_star_placeholder(LocalColumnId column_id) const {

        return std::ranges::find(count_star_placeholder_columns, column_id) !=

               count_star_placeholder_columns.end();

    }

    bool is_predicate_only(LocalColumnId column_id) const {

        return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end();

    }

    LocalIndex non_predicate_position(LocalColumnId column_id) const {

        const auto it = non_predicate_positions.find(column_id);

        return it == non_predicate_positions.end() ? local_positions.at(column_id) : it->second;

    }

    bool has_deferred_non_predicate_column(LocalColumnId column_id) const {

        return non_predicate_positions.contains(column_id);

    }

    size_t block_column_count() const {

        size_t count = 0;

        for (const auto& [_, position] : local_positions) {

            count = std::max(count, position.value() + 1);

        }

        for (const auto& [_, position] : non_predicate_positions) {

            count = std::max(count, position.value() + 1);

        }

        return count;

    }

    // Ingestion rejects malformed records instead of aborting a whole file.

    bool reject_invalid_rows = false;

};

// Helper for constructing the scan-column layout in FileScanRequest.

// FileScanRequest keeps predicate and non-predicate columns separate because columnar readers such

// as Parquet can read predicate columns first, filter rows, and then lazily read the remaining

// projected columns. The two lists still share one file-local output block, whose positions are

// stored in local_positions. This builder centralizes the mechanical rules for that shared layout:

// - each root file column gets one stable predicate block position;

// - predicate columns dominate non-predicate columns because they are already returned in the file

//   block and can be reused for final materialization;

// - a smaller complex predicate subtree may get a second deferred output position;

// - repeated nested projections for the same root are merged instead of duplicated.

// TableColumnMapper should still own table-to-file semantic resolution. This helper only owns the

// FileScanRequest layout contract after a file-local projection has been produced.

class FileScanRequestBuilder {

public:

    explicit FileScanRequestBuilder(FileScanRequest* request) : _request(request) {

        DORIS_CHECK(_request != nullptr);

    }

    Status add_predicate_column(LocalColumnIndex projection) {

        return _add_column(std::move(projection), &_request->predicate_columns,

                           /*is_predicate_column=*/true);

    }

    Status add_non_predicate_column(LocalColumnIndex projection) {

        return _add_column(std::move(projection), &_request->non_predicate_columns,

                           /*is_predicate_column=*/false);

    }

    Status add_deferred_non_predicate_column(LocalColumnIndex projection) {

        const auto file_column_id = projection.column_id();

        DORIS_CHECK(file_column_id != LocalColumnId::invalid());

        DORIS_CHECK(_request->local_positions.contains(file_column_id));

        DORIS_CHECK(std::ranges::any_of(_request->predicate_columns,

                                        [&](const LocalColumnIndex& predicate) {

                                            return predicate.column_id() == file_column_id;

                                        }));

        if (!_request->non_predicate_positions.contains(file_column_id)) {

            _request->non_predicate_positions.emplace(file_column_id,

                                                      _next_block_position(*_request));

        }

        _sort_projection_children_by_file_id(&projection);

        auto existing = std::ranges::find_if(_request->non_predicate_columns,

                                             [&](const LocalColumnIndex& output) {

                                                 return output.column_id() == file_column_id;

                                             });

        if (existing == _request->non_predicate_columns.end()) {

            _request->non_predicate_columns.push_back(std::move(projection));

        } else {

            RETURN_IF_ERROR(merge_local_column_index(&*existing, projection));

            _sort_projection_children_by_file_id(&*existing);

        }

        if (!_request->is_predicate_only(file_column_id)) {

            // The eager complex value has a different physical shape from the final value and

            // must never leak into table materialization after its predicates have run.

            _request->predicate_only_columns.push_back(file_column_id);

        }

        return Status::OK();

    }

    Status add_predicate_column(LocalColumnId column_id) {

        return add_predicate_column(LocalColumnIndex::top_level(column_id));

    }

    Status add_non_predicate_column(LocalColumnId column_id) {

        return add_non_predicate_column(LocalColumnIndex::top_level(column_id));

    }

private:

    static LocalIndex _next_block_position(const FileScanRequest& request) {

        size_t next_position = 0;

        for (const auto& [_, block_position] : request.local_positions) {

            next_position = std::max(next_position, block_position.value() + 1);

        }

        for (const auto& [_, block_position] : request.non_predicate_positions) {

            next_position = std::max(next_position, block_position.value() + 1);

        }

        return LocalIndex(next_position);

    }

    static void _sort_projection_children_by_file_id(LocalColumnIndex* projection) {

        DORIS_CHECK(projection != nullptr);

        if (projection->project_all_children) {

            return;

        }

        for (auto& child : projection->children) {

            _sort_projection_children_by_file_id(&child);

        }

        std::ranges::sort(projection->children,

                          [](const LocalColumnIndex& lhs, const LocalColumnIndex& rhs) {

                              return lhs.local_id() < rhs.local_id();

                          });

    }

    Status _add_column(LocalColumnIndex projection, std::vector<LocalColumnIndex>* scan_columns,

                       bool is_predicate_column) {

        DORIS_CHECK(scan_columns != nullptr);

        const auto file_column_id = projection.column_id();

        DORIS_CHECK(file_column_id != LocalColumnId::invalid());

        if (!is_predicate_column &&

            std::ranges::find_if(_request->predicate_columns,

                                 [&](const LocalColumnIndex& p) {

                                     return p.column_id() == file_column_id;

                                 }) != _request->predicate_columns.end() &&

            !_request->has_deferred_non_predicate_column(file_column_id)) {

            return Status::OK();

        }

        if (!_request->local_positions.contains(file_column_id)) {

            _request->local_positions.emplace(file_column_id, _next_block_position(*_request));

        }

        _sort_projection_children_by_file_id(&projection);

        auto existing_projection_it = std::ranges::find_if(

                *scan_columns,

                [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; });

        if (existing_projection_it == scan_columns->end()) {

            scan_columns->push_back(std::move(projection));

        } else {

            RETURN_IF_ERROR(merge_local_column_index(&*existing_projection_it, projection));

            _sort_projection_children_by_file_id(&*existing_projection_it);

        }

        if (is_predicate_column && !_request->has_deferred_non_predicate_column(file_column_id)) {

            auto it = std::ranges::find_if(

                    _request->non_predicate_columns,

                    [&](const LocalColumnIndex& p) { return p.column_id() == file_column_id; });

            if (it != _request->non_predicate_columns.end()) {

                _request->non_predicate_columns.erase(it);

            }

        }

        return Status::OK();

    }

    FileScanRequest* _request = nullptr;

};

struct FileAggregateRequest {

    struct Column {

        // File-local projection for the aggregate column. For nested MIN/MAX, this points to the

        // single primitive leaf that can be represented by file statistics. For COUNT(col), this

        // points to the top-level column whose NULL-ness should be counted.

        LocalColumnIndex projection;

    };

    TPushAggOp::type agg_type = TPushAggOp::type::NONE;

    // Empty for COUNT(*)/row-count pushdown. Non-empty for COUNT(col), where the file reader must

    // return the number of non-NULL rows for the requested column instead of total rows.

    std::vector<Column> columns;

};

struct FileAggregateResult {

    struct Column {

        // Mirrors FileAggregateRequest::Column::projection so TableReader can put the returned

        // aggregate value back into the matching projected nested shape.

        LocalColumnIndex projection;

        bool has_min = false;

        bool has_max = false;

        Field min_value;

        Field max_value;

    };

    int64_t count = 0;

    std::vector<Column> columns;

};

/**

 *                                +-----> get_schema() -----------------+

 * FileReader() -----> init() ----|                                      -----> close()

 *                                +-----> open() -----> get_block() ----+

 */

class FileReader {

public:

    struct ReaderStatistics {

        int32_t filtered_row_groups = 0;

        int32_t filtered_row_groups_by_min_max = 0;

        int32_t filtered_row_groups_by_bloom_filter = 0;

        int32_t read_row_groups = 0;

        int64_t filtered_group_rows = 0;

        int64_t filtered_page_rows = 0;

        int64_t lazy_read_filtered_rows = 0;

        int64_t read_rows = 0;

        int64_t filtered_bytes = 0;

        int64_t column_read_time = 0;

        int64_t parse_meta_time = 0;

        int64_t parse_footer_time = 0;

        int64_t file_footer_read_calls = 0;

        int64_t file_footer_hit_cache = 0;

        int64_t file_reader_create_time = 0;

        int64_t open_file_num = 0;

        int64_t row_group_filter_time = 0;

        int64_t page_index_filter_time = 0;

        int64_t read_page_index_time = 0;

        int64_t parse_page_index_time = 0;

        int64_t predicate_filter_time = 0;

        int64_t dict_filter_rewrite_time = 0;

        int64_t bloom_filter_read_time = 0;

    };

    FileReader(std::shared_ptr<io::FileSystemProperties>& system_properties,

               std::unique_ptr<io::FileDescription>& file_description,

               std::shared_ptr<io::IOContext> io_ctx, RuntimeProfile* profile)

            : _system_properties(system_properties),

              _file_description(std::move(file_description)),

              _io_ctx(io_ctx),

              _profile(profile) {}

    virtual ~FileReader() = default;

    // Per-batch presence metadata distinguishes absent fields from explicit NULLs. These are

    // physical field identities; TableColumnMapper owns their translation to table positions.

    const std::map<LocalColumnId, std::vector<uint8_t>>& missing_fields() const {

        return _missing_fields;

    }

    // Initialize file reader and parse file metadata.

    virtual Status init(RuntimeState* state);

    // Set the maximum row count for the next physical read batch. Readers that do not batch by

    // rows may ignore it.

    virtual void set_batch_size(size_t batch_size) { (void)batch_size; }

    // Get semantic file-local schema from file metadata. The file schema is determined by file

    // format and file content, and does not contain table/global schema semantics. A file reader may

    // expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier,

    // but it must not interpret table-format semantics such as Iceberg name mapping,

    // default/generated columns, or partition columns. File-format physical wrappers should be

    // normalized away before exposing this schema; for example, Parquet MAP is exposed as key/value

    // children rather than key_value/entry.

    // Doris plans external-table scan types as nullable, including all nested children of complex

    // types. This protects Doris from illegal or inconsistent values produced by external systems.

    // Therefore every ColumnDefinition::type returned here must be nullable. Complex types must

    // also expose nullable child types recursively, even if the physical file marks those fields as

    // required.

    // This method can only be called after init() successfully, but does not require open() to be

    // called.

    virtual Status get_schema(std::vector<ColumnDefinition>* file_schema) const = 0;

    // Create the mapper that matches this reader's scan-request capabilities. TableReader still

    // owns table-format semantics such as BY_NAME/BY_FIELD_ID/BY_INDEX, partition values and

    // default expressions; the FileReader only chooses whether file-local requests support columnar

    // lazy materialization/pruning or must materialize one flat list of required columns.

    virtual std::unique_ptr<TableColumnMapper> create_column_mapper(

            TableColumnMapperOptions options) const;

    virtual bool supports_rowid_fetch() const { return false; }

    // Open the file reader with file-local scan request. The file reader should initialize its internal state according to the request, but does not need to interpret table/global schema semantics. For example, all schema change, filter localization, default/generated/partition columns should be handled in table reader layer. This method can only be called after init() successfully.

    virtual Status open(std::shared_ptr<FileScanRequest> request) {

        if (request->row_ids.has_value()) {

            if (!supports_rowid_fetch()) {

                return Status::NotSupported("File reader does not support row-id fetch");

            }

            const auto& row_ids = *request->row_ids;

            if (std::ranges::any_of(row_ids, [](int64_t row_id) { return row_id < 0; }) ||

                std::ranges::adjacent_find(row_ids, std::greater_equal<>()) != row_ids.end()) {

                return Status::InvalidArgument(

                        "Row-id fetch requires non-negative, strictly increasing file row ids");

            }

        }

        _request = std::move(request);

        return Status::OK();

    }

    // Readers opt in only when they can keep an immutable request for the active physical

    // granule and switch a newer snapshot at a well-defined boundary.

    virtual bool supports_scan_request_refresh() const { return false; }

    virtual Status queue_scan_request(std::shared_ptr<FileScanRequest> request) {

        (void)request;

        return Status::NotSupported("FileReader does not support scan request refresh");

    }

    virtual Status get_block(Block* file_block, size_t* rows, bool* eof) {

        if (rows != nullptr) {

            *rows = 0;

        }

        if (eof != nullptr) {

            *eof = true;

        }

        _eof = true;

        return Status::OK();

    }

    virtual Status get_aggregate_result(const FileAggregateRequest& request,

                                        FileAggregateResult* result) {

        return Status::NotSupported("FileReader does not support aggregate pushdown");

    }

    // Condition cache is managed by TableReader and consumed by physical file readers.

    // On cache HIT, readers may skip granules whose cached bit is false before doing column IO.

    // On cache MISS, readers mark a granule true when row-level predicates keep at least one row

    // in that granule. Readers that cannot map batch rows to stable file-global row ids should

    // keep the default no-op implementation.

    virtual void set_condition_cache_context(std::shared_ptr<ConditionCacheContext> ctx) {}

    // Total rows covered by this physical reader. TableReader uses it to pre-size the miss bitmap.

    // Readers should return 0 if the metadata is unavailable or the row coordinate is unstable.

    virtual int64_t get_total_rows() const { return 0; }

    virtual Status close() {

        _file_reader.reset();

        _tracing_file_reader.reset();

        _io_ctx.reset();

        _eof = true;

        return Status::OK();

    }

protected:

    virtual void _init_profile() {}

    void _record_scan_rows(int64_t rows) {

        DORIS_CHECK(rows >= 0);

        _reader_statistics.read_rows += rows;

        if (_io_ctx != nullptr && _io_ctx->file_reader_stats != nullptr) {

            _io_ctx->file_reader_stats->read_rows += cast_set<size_t>(rows);

        }

    }

    io::FileReaderSPtr _file_reader;

    // _tracing_file_reader wraps _file_reader.

    // _file_reader is original file reader.

    // _tracing_file_reader is tracing file reader with io context.

    // If io_ctx is null, _tracing_file_reader will be the same as file_reader.

    io::FileReaderSPtr _tracing_file_reader = nullptr;

    std::shared_ptr<FileScanRequest> _request;

    bool _eof = true;

    ReaderStatistics _reader_statistics;

    std::shared_ptr<io::FileSystemProperties> _system_properties;

    std::unique_ptr<io::FileDescription> _file_description;

    std::shared_ptr<io::IOContext> _io_ctx;

    RuntimeProfile* _profile = nullptr;

    std::map<LocalColumnId, std::vector<uint8_t>> _missing_fields;

};

} // namespace doris::format