Coverage Report

Created: 2024-11-18 10:37

/root/doris/be/src/vec/jsonb/serialize.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "vec/jsonb/serialize.h"
19
20
#include <assert.h>
21
22
#include <algorithm>
23
#include <memory>
24
#include <vector>
25
26
#include "olap/tablet_schema.h"
27
#include "runtime/descriptors.h"
28
#include "runtime/jsonb_value.h"
29
#include "runtime/primitive_type.h"
30
#include "runtime/types.h"
31
#include "util/bitmap_value.h"
32
#include "util/jsonb_document.h"
33
#include "util/jsonb_stream.h"
34
#include "util/jsonb_writer.h"
35
#include "vec/columns/column.h"
36
#include "vec/columns/column_string.h"
37
#include "vec/common/arena.h"
38
#include "vec/common/string_ref.h"
39
#include "vec/core/column_with_type_and_name.h"
40
#include "vec/core/columns_with_type_and_name.h"
41
#include "vec/data_types/data_type.h"
42
#include "vec/data_types/serde/data_type_serde.h"
43
#include "vec/io/reader_buffer.h"
44
45
namespace doris::vectorized {
46
47
void JsonbSerializeUtil::block_to_jsonb(const TabletSchema& schema, const Block& block,
48
                                        ColumnString& dst, int num_cols,
49
4
                                        const DataTypeSerDeSPtrs& serdes) {
50
4
    auto num_rows = block.rows();
51
4
    Arena pool;
52
4
    assert(num_cols <= block.columns());
53
1.03k
    for (int i = 0; i < num_rows; ++i) {
54
1.03k
        JsonbWriterT<JsonbOutStream> jsonb_writer;
55
1.03k
        jsonb_writer.writeStartObject();
56
10.2k
        for (int j = 0; j < num_cols; ++j) {
57
9.22k
            const auto& column = block.get_by_position(j).column;
58
9.22k
            const auto& tablet_column = schema.columns()[j];
59
9.22k
            if (tablet_column.is_row_store_column()) {
60
                // ignore dst row store column
61
0
                continue;
62
0
            }
63
9.22k
            serdes[j]->write_one_cell_to_jsonb(*column, jsonb_writer, &pool,
64
9.22k
                                               tablet_column.unique_id(), i);
65
9.22k
        }
66
1.03k
        jsonb_writer.writeEndObject();
67
1.03k
        dst.insert_data(jsonb_writer.getOutput()->getBuffer(), jsonb_writer.getOutput()->getSize());
68
1.03k
    }
69
4
}
70
71
// batch rows
72
void JsonbSerializeUtil::jsonb_to_block(const DataTypeSerDeSPtrs& serdes,
73
                                        const ColumnString& jsonb_column,
74
                                        const std::unordered_map<uint32_t, uint32_t>& col_id_to_idx,
75
                                        Block& dst,
76
4
                                        const std::vector<std::string>& default_values) {
77
1.03k
    for (int i = 0; i < jsonb_column.size(); ++i) {
78
1.03k
        StringRef jsonb_data = jsonb_column.get_data_at(i);
79
1.03k
        jsonb_to_block(serdes, jsonb_data.data, jsonb_data.size, col_id_to_idx, dst,
80
1.03k
                       default_values);
81
1.03k
    }
82
4
}
83
84
// single row
85
void JsonbSerializeUtil::jsonb_to_block(const DataTypeSerDeSPtrs& serdes, const char* data,
86
                                        size_t size,
87
                                        const std::unordered_map<uint32_t, uint32_t>& col_id_to_idx,
88
                                        Block& dst,
89
1.03k
                                        const std::vector<std::string>& default_values) {
90
1.03k
    auto pdoc = JsonbDocument::createDocument(data, size);
91
1.03k
    JsonbDocument& doc = *pdoc;
92
1.03k
    size_t num_rows = dst.rows();
93
1.03k
    size_t filled_columns = 0;
94
10.2k
    for (auto it = doc->begin(); it != doc->end(); ++it) {
95
9.22k
        auto col_it = col_id_to_idx.find(it->getKeyId());
96
9.22k
        if (col_it != col_id_to_idx.end()) {
97
9.22k
            MutableColumnPtr dst_column =
98
9.22k
                    dst.get_by_position(col_it->second).column->assume_mutable();
99
9.22k
            serdes[col_it->second]->read_one_cell_from_jsonb(*dst_column, it->value());
100
9.22k
            ++filled_columns;
101
9.22k
        }
102
9.22k
    }
103
1.03k
    if (filled_columns < dst.columns()) {
104
        // fill missing slot
105
0
        for (int i = 0; i < dst.columns(); ++i) {
106
0
            const auto& column_type_name = dst.get_by_position(i);
107
0
            MutableColumnPtr col = column_type_name.column->assume_mutable();
108
0
            if (col->size() < num_rows + 1) {
109
0
                DCHECK(col->size() == num_rows);
110
0
                if (default_values[i].empty()) {
111
0
                    col->insert_default();
112
0
                } else {
113
0
                    Slice value(default_values[i].data(), default_values[i].size());
114
0
                    DataTypeSerDe::FormatOptions opt;
115
0
                    opt.converted_from_string = true;
116
0
                    static_cast<void>(serdes[i]->deserialize_one_cell_from_json(*col, value, opt));
117
0
                }
118
0
            }
119
0
            DCHECK(col->size() == num_rows + 1);
120
0
        }
121
0
    }
122
1.03k
}
123
124
} // namespace doris::vectorized