Coverage Report

Created: 2026-10-31 16:54

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/format/table/iceberg/schema_parser.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "format/table/iceberg/schema_parser.h"
19
20
#include <rapidjson/document.h>
21
#include <rapidjson/error/en.h>
22
23
#include <cstdint>
24
#include <optional>
25
#include <unordered_set>
26
27
#include "format/table/iceberg/schema.h"
28
#include "format/table/iceberg/types.h"
29
30
namespace doris::iceberg {
31
32
const char* SchemaParser::SCHEMA_ID = "schema-id";
33
const char* SchemaParser::IDENTIFIER_FIELD_IDS = "identifier-field-ids";
34
const char* SchemaParser::TYPE = "type";
35
const char* SchemaParser::STRUCT = "struct";
36
const char* SchemaParser::LIST = "list";
37
const char* SchemaParser::MAP = "map";
38
const char* SchemaParser::FIELDS = "fields";
39
const char* SchemaParser::ELEMENT = "element";
40
const char* SchemaParser::KEY = "key";
41
const char* SchemaParser::VALUE = "value";
42
const char* SchemaParser::DOC = "doc";
43
const char* SchemaParser::NAME = "name";
44
const char* SchemaParser::ID = "id";
45
const char* SchemaParser::ELEMENT_ID = "element-id";
46
const char* SchemaParser::KEY_ID = "key-id";
47
const char* SchemaParser::VALUE_ID = "value-id";
48
const char* SchemaParser::REQUIRED = "required";
49
const char* SchemaParser::ELEMENT_REQUIRED = "element-required";
50
const char* SchemaParser::VALUE_REQUIRED = "value-required";
51
52
777
std::unique_ptr<Type> SchemaParser::_type_from_json(const rapidjson::Value& value) {
53
777
    if (value.IsString()) {
54
        // Opaque CRS text can contain a decoded NUL: geography(custom)\0zone, spherical)
55
        // must not become the different, valid geography(custom) before type parsing.
56
568
        return Types::from_primitive_string(
57
568
                std::string(value.GetString(), value.GetStringLength()));
58
568
    } else if (value.IsObject()) {
59
209
        if (value.HasMember(TYPE)) {
60
209
            std::string type = value[TYPE].GetString();
61
209
            if (type == STRUCT) {
62
161
                return _struct_from_json(value);
63
161
            } else if (type == LIST) {
64
25
                return _list_from_json(value);
65
25
            } else if (type == MAP) {
66
23
                return _map_from_json(value);
67
23
            }
68
209
        }
69
209
    }
70
0
    throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Cannot parse type from json.");
71
777
}
72
73
161
std::unique_ptr<StructType> SchemaParser::_struct_from_json(const rapidjson::Value& value) {
74
161
    if (!value.HasMember("fields") || !value["fields"].IsArray()) {
75
0
        throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
76
0
                               "Cannot parse struct fields from non-array.");
77
0
    }
78
79
161
    const rapidjson::Value& field_array = value["fields"];
80
161
    std::vector<NestedField> fields;
81
161
    fields.reserve(field_array.Size());
82
83
712
    for (uint32_t i = 0; i < field_array.Size(); ++i) {
84
551
        const rapidjson::Value& field = field_array[i];
85
551
        if (!field.IsObject()) {
86
0
            throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
87
0
                                   "Cannot parse struct field from non-object.");
88
0
        }
89
90
551
        int id = field[ID].GetInt();
91
551
        std::string name = field[NAME].GetString();
92
551
        std::unique_ptr<Type> type = _type_from_json(field[TYPE]);
93
94
551
        std::optional<std::string> doc = std::nullopt;
95
551
        if (field.HasMember(DOC)) {
96
1
            doc = field[DOC].GetString();
97
1
        }
98
99
551
        bool is_required = field[REQUIRED].GetBool();
100
101
551
        fields.emplace_back(!is_required, id, name, std::move(type), doc);
102
551
    }
103
104
161
    return std::make_unique<StructType>(std::move(fields));
105
161
}
106
107
25
std::unique_ptr<ListType> SchemaParser::_list_from_json(const rapidjson::Value& value) {
108
25
    int element_id = value[ELEMENT_ID].GetInt();
109
25
    std::unique_ptr<Type> element_type = _type_from_json(value[ELEMENT]);
110
25
    bool is_required = value[ELEMENT_REQUIRED].GetBool();
111
112
25
    if (is_required) {
113
7
        return ListType::of_required(element_id, std::move(element_type));
114
18
    } else {
115
18
        return ListType::of_optional(element_id, std::move(element_type));
116
18
    }
117
25
}
118
119
23
std::unique_ptr<MapType> SchemaParser::_map_from_json(const rapidjson::Value& value) {
120
23
    int key_id = value[KEY_ID].GetInt();
121
23
    std::unique_ptr<Type> key_type = _type_from_json(value[KEY]);
122
123
23
    int value_id = value[VALUE_ID].GetInt();
124
23
    std::unique_ptr<Type> value_type = _type_from_json(value[VALUE]);
125
126
23
    bool is_required = value[VALUE_REQUIRED].GetBool();
127
128
23
    if (is_required) {
129
7
        return MapType::of_required(key_id, value_id, std::move(key_type), std::move(value_type));
130
16
    } else {
131
16
        return MapType::of_optional(key_id, value_id, std::move(key_type), std::move(value_type));
132
16
    }
133
23
}
134
135
148
std::unique_ptr<Schema> SchemaParser::from_json(const std::string& json) {
136
148
    rapidjson::Document doc;
137
148
    doc.Parse(json.c_str());
138
139
148
    if (doc.HasParseError()) {
140
0
        throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Failed to parse JSON: {}.",
141
0
                               std::string(GetParseError_En(doc.GetParseError())));
142
0
    }
143
148
    std::unique_ptr<Type> type = _type_from_json(doc);
144
148
    return std::make_unique<Schema>(type->as_nested_type()->as_struct_type()->move_fields());
145
148
}
146
147
std::unordered_set<int> SchemaParser::_get_integer_set(const char* key,
148
0
                                                       const rapidjson::Value& value) {
149
0
    std::unordered_set<int> integer_set;
150
151
0
    if (value.HasMember(key) && value[key].IsArray()) {
152
0
        const rapidjson::Value& arr = value[key];
153
0
        for (uint32_t i = 0; i < arr.Size(); i++) {
154
0
            if (arr[i].IsInt()) {
155
0
                integer_set.insert(arr[i].GetInt());
156
0
            } else {
157
0
                throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
158
0
                                       "Unexpected non-integer element in the array.");
159
0
            }
160
0
        }
161
0
    }
162
0
    return integer_set;
163
0
}
164
165
} // namespace doris::iceberg