// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//

//   http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.

#include <glog/logging.h>

#include <stddef.h>

#include <memory>

#include <ostream>

#include <span>

#include <string>

#include <string_view>

#include <utility>

#include <vector>

#include "common/status.h"

#include "core/assert_cast.h"

#include "core/block/block.h"

#include "core/column/column.h"

#include "core/column/column_nullable.h"

#include "core/column/column_string.h"

#include "core/column/column_variant.h"

#include "core/column/subcolumn_tree.h"

#include "core/column/variant_column_utils.h"

#include "core/column/variant_v2/column_variant_v2.h"

#include "core/data_type/data_type.h"

#include "core/data_type/data_type_nothing.h"

#include "core/data_type/data_type_nullable.h"

#include "core/data_type/data_type_number.h"

#include "core/data_type/data_type_string.h"

#include "core/data_type/data_type_variant.h"

#include "core/string_ref.h"

#include "exprs/function/function.h"

#include "exprs/function/function_helpers.h"

#include "exprs/function/function_variant_element_v2.h"

#include "exprs/function/simple_function_factory.h"

#include "exprs/json_functions.h"

#include "simdjson.h"

#include "util/defer_op.h"

#include "util/json/path_in_data.h"

namespace doris {

class FunctionVariantElement : public IFunction {

public:

    static constexpr auto name = "element_at";

    static FunctionPtr create() { return std::make_shared<FunctionVariantElement>(); }

    // Get function name.

    String get_name() const override { return name; }

    bool use_default_implementation_for_nulls() const override { return false; }

    size_t get_number_of_arguments() const override { return 2; }

    ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }

    DataTypes get_variadic_argument_types_impl() const override {

        return {std::make_shared<DataTypeVariant>(), std::make_shared<DataTypeString>()};

    }

    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {

        DCHECK_EQ(arguments[0]->get_primitive_type(), TYPE_VARIANT)

                << "First argument for function: " << name

                << " should be DataTypeVariant but it has type " << arguments[0]->get_name() << ".";

        const PrimitiveType index_type = remove_nullable(arguments[1])->get_primitive_type();

        DCHECK(is_string_type(index_type) || is_int_or_bool(index_type))

                << "Second argument for function: " << name

                << " should be String or Integer but it has type " << arguments[1]->get_name()

                << ".";

        auto arg_variant = remove_nullable(arguments[0]);

        return make_nullable(std::move(arg_variant));

    }

    // wrap variant column with nullable

    // 1. if variant is null root(empty or nothing as root), then nullable map is all null

    // 2. if variant is scalar variant, then use the root's nullable map

    // 3. if variant is hierarchical variant, then create a nullable map with all none null

    ColumnPtr wrap_variant_nullable(ColumnPtr col) const {

        const auto& var = assert_cast<const ColumnVariant&>(*col);

        if (var.is_null_root()) {

            return make_nullable(col, true);

        }

        if (var.is_scalar_variant() && is_column_nullable(*var.get_root())) {

            const auto* nullable = assert_cast<const ColumnNullable*>(var.get_root().get());

            return ColumnNullable::create(col, nullable->get_null_map_column_ptr());

        }

        return make_nullable(col);

    }

    // Keep legacy/V2 physical-column dispatch in one entry point so nullable handling stays shared.

    // NOLINTNEXTLINE(readability-function-size)

    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,

                        uint32_t result, size_t input_rows_count) const override {

        const ColumnPtr materialized =

                block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();

        const IColumn* physical = materialized.get();

        std::span<const uint8_t> outer_nulls;

        if (const auto* nullable = check_and_get_column<ColumnNullable>(physical)) {

            outer_nulls = nullable->get_null_map_data();

            physical = &nullable->get_nested_column();

        }

        if (const auto* variant_v2 = check_and_get_column<ColumnVariantV2>(physical)) {

            if (block.empty()) {

                block.replace_by_position(result, ColumnNullable::create(ColumnVariantV2::create(),

                                                                         ColumnUInt8::create()));

                return Status::OK();

            }

            auto replace_with_all_null_result = [&]() {

                auto null_values = ColumnVariantV2::create();

                null_values->insert_many_defaults(variant_v2->size());

                block.replace_by_position(

                        result, ColumnNullable::create(std::move(null_values),

                                                       ColumnUInt8::create(variant_v2->size(), 1)));

            };

            const auto& index_argument = block.get_by_position(arguments[1]);

            const ColumnPtr materialized_index =

                    index_argument.column->convert_to_full_column_if_const();

            const IColumn* index_column = materialized_index.get();

            if (index_column->is_null_at(0)) {

                replace_with_all_null_result();

                return Status::OK();

            }

            if (const auto* nullable = check_and_get_column<ColumnNullable>(*index_column)) {

                index_column = &nullable->get_nested_column();

            }

            std::optional<VariantElementV2PathSegment> segment;

            const PrimitiveType index_type =

                    remove_nullable(index_argument.type)->get_primitive_type();

            if (is_string_type(index_type)) {

                segment = VariantElementV2PathSegment::object_key(index_column->get_data_at(0));

            } else if (is_int_or_bool(index_type)) {

                const int64_t sql_index = index_column->get_int(0);

                if (sql_index == 0) {

                    replace_with_all_null_result();

                    return Status::OK();

                }

                segment = VariantElementV2PathSegment::array_index(sql_index > 0 ? sql_index - 1

                                                                                 : sql_index);

            } else {

                return Status::RuntimeError("unsupported index type {} for function {}",

                                            index_argument.type->get_name(), get_name());

            }

            std::unique_ptr<ResolvedVariantElementV2Path> path;

            RETURN_IF_ERROR(resolve_variant_element_v2_path(std::span(&*segment, 1), &path));

            ColumnPtr result_column;

            RETURN_IF_ERROR(

                    extract_variant_element_v2(*variant_v2, *path, outer_nulls, &result_column));

            block.replace_by_position(result, std::move(result_column));

            return Status::OK();

        }

        const auto* variant_col =

                check_and_get_column<ColumnVariant>(remove_nullable(materialized).get());

        if (!variant_col) {

            return Status::RuntimeError(

                    fmt::format("unsupported types for function {}({}, {})", get_name(),

                                block.get_by_position(arguments[0]).type->get_name(),

                                block.get_by_position(arguments[1]).type->get_name()));

        }

        if (block.empty()) {

            block.replace_by_position(result, block.get_by_position(result).type->create_column());

            return Status::OK();

        }

        auto index_column = block.get_by_position(arguments[1]).column;

        ColumnPtr result_column;

        RETURN_IF_ERROR(get_element_column(*variant_col, index_column, &result_column));

        if (block.get_by_position(result).type->is_nullable()) {

            result_column = wrap_variant_nullable(result_column);

        }

        block.replace_by_position(result, result_column);

        return Status::OK();

    }

private:

    // Return sub-path by specified prefix.

    // For example, for prefix a.b:

    // a.b.c.d -> c.d, a.b.c -> c

    static std::optional<std::string_view> get_sub_path(const std::string_view& path,

                                                        const std::string_view& prefix) {

        if (path.size() <= prefix.size() || path[prefix.size()] != '.') {

            return std::nullopt;

        }

        return path.substr(prefix.size() + 1);

    }

    // Extract and populate sparse column data with given path prefix

    // Copies data from source sparse column, extracting only the sub-paths that match the prefix

    static void _extract_sparse_column_from_source(ColumnVariant* src_ptr, const PathInData& path,

                                                   ColumnVariant::MutablePtr& target_ptr) {

        ColumnVariant::Subcolumn root {0, true, true};

        // no root, no sparse column

        const auto& sparse_data_map = assert_cast<const ColumnMap&>(*src_ptr->get_sparse_column());

        const auto& src_sparse_data_offsets = sparse_data_map.get_offsets();

        const auto& src_sparse_data_paths =

                assert_cast<const ColumnString&>(sparse_data_map.get_keys());

        const auto& src_sparse_data_values =

                assert_cast<const ColumnString&>(sparse_data_map.get_values());

        auto& sparse_data_offsets =

                assert_cast<ColumnMap&>(target_ptr->get_sparse_column_mutable()).get_offsets();

        auto [sparse_data_paths, sparse_data_values] =

                target_ptr->get_sparse_data_paths_and_values();

        StringRef prefix_ref(path.get_path());

        std::string_view path_prefix(prefix_ref.data, prefix_ref.size);

        for (size_t i = 0; i != src_sparse_data_offsets.size(); ++i) {

            size_t start = src_sparse_data_offsets[ssize_t(i) - 1];

            size_t end = src_sparse_data_offsets[ssize_t(i)];

            size_t lower_bound_index = find_variant_sparse_path_lower_bound(

                    prefix_ref, src_sparse_data_paths, start, end);

            for (; lower_bound_index != end; ++lower_bound_index) {

                auto path_ref = src_sparse_data_paths.get_data_at(lower_bound_index);

                std::string_view nested_path(path_ref.data, path_ref.size);

                if (!nested_path.starts_with(path_prefix)) {

                    break;

                }

                // Don't include path that is equal to the prefix.

                if (nested_path.size() != path_prefix.size()) {

                    auto sub_path_optional = get_sub_path(nested_path, path_prefix);

                    if (!sub_path_optional.has_value()) {

                        continue;

                    }

                    std::string_view sub_path = *sub_path_optional;

                    sparse_data_paths->insert_data(sub_path.data(), sub_path.size());

                    sparse_data_values->insert_from(src_sparse_data_values, lower_bound_index);

                } else {

                    // insert into root column, example:  access v['b'] and b is in sparse column

                    // data example:

                    // {"b" : 123}

                    // {"b" : {"c" : 456}}

                    // b maybe in sparse column, and b.c is in subolumn, put `b` into root column to distinguish

                    // from "" which is empty path and root

                    root.deserialize_from_binary_column(&src_sparse_data_values, lower_bound_index);

                }

            }

            if (root.size() == sparse_data_offsets.size()) {

                root.insert_default();

            }

            sparse_data_offsets.push_back(sparse_data_paths->size());

        }

        target_ptr->get_subcolumns().create_root(root);

        target_ptr->get_doc_value_column_mutable().resize(src_ptr->size());

        target_ptr->set_num_rows(src_ptr->size());

    }

    // Extract and populate sparse column data from doc_value column with given path prefix

    // Copies data from source doc_value column, extracting only the sub-paths that match the prefix

    static void _extract_doc_value_column_from_source(ColumnVariant* src_ptr,

                                                      const PathInData& path,

                                                      ColumnVariant::MutablePtr& target_ptr) {

        ColumnVariant::Subcolumn root {0, true, true};

        const auto& doc_value_data_map =

                assert_cast<const ColumnMap&>(*src_ptr->get_doc_value_column());

        const auto& src_doc_value_data_offsets = doc_value_data_map.get_offsets();

        const auto& src_doc_value_data_paths =

                assert_cast<const ColumnString&>(doc_value_data_map.get_keys());

        const auto& src_doc_value_data_values =

                assert_cast<const ColumnString&>(doc_value_data_map.get_values());

        const bool write_to_doc_value = target_ptr->enable_doc_mode();

        // Ordinary Variant extraction keeps the selected prefix in sparse data, matching the

        // source branch behavior. Only doc-mode columns keep extracted data in doc_value.

        auto& extracted_offsets =

                assert_cast<ColumnMap&>(write_to_doc_value

                                                ? target_ptr->get_doc_value_column_mutable()

                                                : target_ptr->get_sparse_column_mutable())

                        .get_offsets();

        auto [extracted_paths, extracted_values] =

                write_to_doc_value ? target_ptr->get_doc_value_data_paths_and_values()

                                   : target_ptr->get_sparse_data_paths_and_values();

        StringRef prefix_ref(path.get_path());

        std::string_view path_prefix(prefix_ref.data, prefix_ref.size);

        for (size_t i = 0; i != src_doc_value_data_offsets.size(); ++i) {

            size_t start = src_doc_value_data_offsets[ssize_t(i) - 1];

            size_t end = src_doc_value_data_offsets[ssize_t(i)];

            size_t lower_bound_index = find_variant_sparse_path_lower_bound(

                    prefix_ref, src_doc_value_data_paths, start, end);

            for (; lower_bound_index != end; ++lower_bound_index) {

                auto path_ref = src_doc_value_data_paths.get_data_at(lower_bound_index);

                std::string_view nested_path(path_ref.data, path_ref.size);

                if (!nested_path.starts_with(path_prefix)) {

                    break;

                }

                if (nested_path.size() != path_prefix.size()) {

                    auto sub_path_optional = get_sub_path(nested_path, path_prefix);

                    if (!sub_path_optional.has_value()) {

                        continue;

                    }

                    std::string_view sub_path = *sub_path_optional;

                    extracted_paths->insert_data(sub_path.data(), sub_path.size());

                    extracted_values->insert_from(src_doc_value_data_values, lower_bound_index);

                } else {

                    root.deserialize_from_binary_column(&src_doc_value_data_values,

                                                        lower_bound_index);

                }

            }

            if (root.size() == extracted_offsets.size()) {

                root.insert_default();

            }

            extracted_offsets.push_back(extracted_paths->size());

        }

        target_ptr->get_subcolumns().create_root(root);

        if (write_to_doc_value) {

            target_ptr->get_sparse_column_mutable().resize(src_ptr->size());

        } else {

            target_ptr->get_doc_value_column_mutable().resize(src_ptr->size());

        }

        target_ptr->set_num_rows(src_ptr->size());

    }

    static Status get_element_column(const ColumnVariant& src, const ColumnPtr& index_column,

                                     ColumnPtr* result) {

        std::string field_name = index_column->get_data_at(0).to_string();

        if (src.empty()) {

            *result = ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode());

            // src subcolumns empty but src row count may not be 0

            (*result)->assert_mutable()->insert_many_defaults(src.size());

            // ColumnVariant should be finalized before parsing, finalize maybe modify original column structure

            (*result)->assert_mutable()->finalize();

            return Status::OK();

        }

        if (src.is_scalar_variant() && is_string_type(src.get_root_type()->get_primitive_type())) {

            // use parser to extract from root

            auto type = std::make_shared<DataTypeString>();

            MutableColumnPtr result_column = type->create_column();

            const ColumnString& docs =

                    *assert_cast<const ColumnString*>(remove_nullable(src.get_root()).get());

            simdjson::ondemand::parser parser;

            std::vector<JsonPath> parsed_paths;

            if (field_name.empty() || field_name[0] != '$') {

                field_name = "$." + field_name;

            }

            JsonFunctions::parse_json_paths(field_name, &parsed_paths);

            ColumnString* col_str = static_cast<ColumnString*>(result_column.get());

            for (size_t i = 0; i < docs.size(); ++i) {

                if (!extract_from_document(parser, docs.get_data_at(i), parsed_paths, col_str)) {

                    VLOG_DEBUG << "failed to parse " << docs.get_data_at(i) << ", field "

                               << field_name;

                    result_column->insert_default();

                }

            }

            *result = ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode(), type,

                                            std::move(result_column));

            (*result)->assert_mutable()->finalize();

            return Status::OK();

        } else {

            auto mutable_src = src.clone_finalized();

            auto* mutable_ptr = assert_cast<ColumnVariant*>(mutable_src.get());

            PathInData path(field_name);

            ColumnVariant::Subcolumns subcolumns = mutable_ptr->get_subcolumns();

            const auto* node = subcolumns.find_exact(path);

            MutableColumnPtr result_col =

                    ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode());

            ColumnVariant::Subcolumns new_subcolumns;

            if (node != nullptr) {

                std::vector<decltype(node)> nodes;

                PathsInData paths;

                ColumnVariant::Subcolumns::get_leaves_of_node(node, nodes, paths);

                for (const auto* n : nodes) {

                    PathInData new_path = n->path.copy_pop_front();

                    VLOG_DEBUG << "add node " << new_path.get_path()

                               << ", data size: " << n->data.size()

                               << ", finalized size: " << n->data.get_finalized_column().size()

                               << ", common type: " << n->data.get_least_common_type()->get_name();

                    // if new_path is empty, indicate it's the root column, but adding a root will return false when calling add

                    if (!new_subcolumns.add(new_path, n->data)) {

                        VLOG_DEBUG << "failed to add node " << new_path.get_path();

                    }

                }

                // handle the root node

                if (new_subcolumns.empty() && !nodes.empty()) {

                    CHECK_EQ(nodes.size(), 1);

                    new_subcolumns.create_root(ColumnVariant::Subcolumn {

                            IColumn::mutate(nodes[0]->data.get_finalized_column_ptr()),

                            nodes[0]->data.get_least_common_type(), true, true});

                    auto container =

                            ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode(),

                                                  std::move(new_subcolumns));

                    result_col->insert_range_from(*container, 0, container->size());

                } else {

                    auto container =

                            ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode(),

                                                  std::move(new_subcolumns));

                    container->clear_sparse_column();

                    _extract_sparse_column_from_source(mutable_ptr, path, container);

                    result_col->insert_range_from(*container, 0, container->size());

                }

            } else {

                auto container =

                        ColumnVariant::create(src.max_subcolumns_count(), src.enable_doc_mode(),

                                              std::move(new_subcolumns));

                const auto& sparse_offsets = mutable_ptr->serialized_sparse_column_offsets();

                if (sparse_offsets.back() == sparse_offsets[-1]) {

                    _extract_doc_value_column_from_source(mutable_ptr, path, container);

                } else {

                    _extract_sparse_column_from_source(mutable_ptr, path, container);

                }

                result_col->insert_range_from(*container, 0, container->size());

            }

            // ColumnVariant should be finalized before parsing, finalize maybe modify original column structure

            result_col->finalize();

            VLOG_DEBUG << "dump new object "

                       << static_cast<const ColumnVariant*>(result_col.get())->debug_string()

                       << ", path " << path.get_path();

            *result = std::move(result_col);

            return Status::OK();

        }

    }

    static Status extract_from_document(simdjson::ondemand::parser& parser, const StringRef& doc,

                                        const std::vector<JsonPath>& paths, ColumnString* column) {

        try {

            simdjson::padded_string json_str {doc.data, doc.size};

            simdjson::ondemand::document document = parser.iterate(json_str);

            simdjson::ondemand::object object = document.get_object();

            simdjson::ondemand::value value;

            RETURN_IF_ERROR(JsonFunctions::extract_from_object(object, paths, &value));

            _write_data_to_column(value, column);

        } catch (simdjson::simdjson_error& e) {

            VLOG_DEBUG << "simdjson parse exception: " << e.what();

            return Status::DataQualityError("simdjson parse exception {}", e.what());

        }

        return Status::OK();

    }

    static void _write_data_to_column(simdjson::ondemand::value& value, ColumnString* column) {

        switch (value.type()) {

        case simdjson::ondemand::json_type::null: {

            column->insert_default();

            break;

        }

        case simdjson::ondemand::json_type::boolean: {

            if (value.get_bool()) {

                column->insert_data("1", 1);

            } else {

                column->insert_data("0", 1);

            }

            break;

        }

        case simdjson::ondemand::json_type::string: {

            // Extract the raw (unescaped) string value rather than its JSON

            // representation. simdjson::to_json_string would keep the surrounding

            // double quotes (e.g. "2026-05-20"), which leaks into the result and

            // makes scalar-string variants inconsistent with structured ones.

            std::string_view value_str = value.get_string().value();

            column->insert_data(value_str.data(), value_str.length());

            break;

        }

        default: {

            auto value_str = simdjson::to_json_string(value).value();

            column->insert_data(value_str.data(), value_str.length());

        }

        }

    }

};

class FunctionVariantElementByInteger final : public FunctionVariantElement {

public:

    static constexpr auto name = FunctionVariantElement::name;

    static FunctionPtr create() { return std::make_shared<FunctionVariantElementByInteger>(); }

    DataTypes get_variadic_argument_types_impl() const override {

        return {std::make_shared<DataTypeVariant>(), std::make_shared<DataTypeInt64>()};

    }

};

void register_function_variant_element(SimpleFunctionFactory& factory) {

    factory.register_function<FunctionVariantElement>();

    factory.register_function<FunctionVariantElementByInteger>();

}

} // namespace doris