Coverage Report

Created: 2026-07-30 16:06

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/format/parquet/parquet_thrift_util.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/parquet_types.h>
21
22
#include <cstdint>
23
24
#include "common/logging.h"
25
#include "common/status.h"
26
#include "core/custom_allocator.h"
27
#include "format/parquet/vparquet_file_metadata.h"
28
#include "io/fs/file_reader.h"
29
#include "io/io_common.h"
30
#include "storage/iterators.h"
31
#include "util/coding.h"
32
#include "util/thrift_util.h"
33
34
namespace doris {
35
constexpr uint8_t PARQUET_VERSION_NUMBER[4] = {'P', 'A', 'R', '1'};
36
constexpr uint32_t PARQUET_FOOTER_SIZE = 8;
37
constexpr size_t INIT_META_SIZE = 48 * 1024; // 48k
38
39
static Status parse_thrift_footer(io::FileReaderSPtr file,
40
                                  std::unique_ptr<FileMetaData>* file_metadata, size_t* meta_size,
41
                                  io::IOContext* io_ctx, const bool enable_mapping_varbinary,
42
134
                                  const bool enable_mapping_timestamp_tz) {
43
134
    size_t file_size = file->size();
44
134
    size_t bytes_read = std::min(file_size, INIT_META_SIZE);
45
134
    std::vector<uint8_t> footer(bytes_read);
46
134
    RETURN_IF_ERROR(file->read_at(file_size - bytes_read, Slice(footer.data(), bytes_read),
47
134
                                  &bytes_read, io_ctx));
48
49
    // validate magic
50
134
    uint8_t* magic_ptr = footer.data() + bytes_read - 4;
51
134
    if (bytes_read < PARQUET_FOOTER_SIZE) {
52
0
        return Status::Corruption(
53
0
                "Read parquet file footer fail, bytes read: {}, file size: {}, path: {}",
54
0
                bytes_read, file_size, file->path().native());
55
134
    } else if (memcmp(magic_ptr, PARQUET_VERSION_NUMBER, sizeof(PARQUET_VERSION_NUMBER)) != 0) {
56
0
        return Status::Corruption(
57
0
                "Invalid magic number in parquet file, bytes read: {}, file size: {}, path: {}, "
58
0
                "read magic: {}",
59
0
                bytes_read, file_size, file->path().native(),
60
0
                std::string((char*)magic_ptr, sizeof(PARQUET_VERSION_NUMBER)));
61
0
    }
62
63
    // get metadata_size
64
134
    uint32_t metadata_size = decode_fixed32_le(footer.data() + bytes_read - PARQUET_FOOTER_SIZE);
65
134
    if (metadata_size > file_size - PARQUET_FOOTER_SIZE) {
66
0
        return Status::Corruption("Parquet footer size({}) is large than file size({})",
67
0
                                  metadata_size, file_size);
68
0
    }
69
134
    DorisUniqueBufferPtr<uint8_t> new_buff;
70
134
    uint8_t* meta_ptr;
71
134
    if (metadata_size > bytes_read - PARQUET_FOOTER_SIZE) {
72
0
        new_buff = make_unique_buffer<uint8_t>(metadata_size);
73
0
        RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size,
74
0
                                      Slice(new_buff.get(), metadata_size), &bytes_read, io_ctx));
75
0
        meta_ptr = new_buff.get();
76
134
    } else {
77
134
        meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size;
78
134
    }
79
80
134
    tparquet::FileMetaData t_metadata;
81
    // deserialize footer
82
134
    RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata));
83
134
    *file_metadata = std::make_unique<FileMetaData>(t_metadata, metadata_size);
84
134
    RETURN_IF_ERROR(
85
134
            (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz));
86
134
    *meta_size = PARQUET_FOOTER_SIZE + metadata_size;
87
134
    return Status::OK();
88
134
}
parquet_expr_test.cpp:_ZN5dorisL19parse_thrift_footerESt10shared_ptrINS_2io10FileReaderEEPSt10unique_ptrINS_12FileMetaDataESt14default_deleteIS5_EEPmPNS1_9IOContextEbb
Line
Count
Source
42
39
                                  const bool enable_mapping_timestamp_tz) {
43
39
    size_t file_size = file->size();
44
39
    size_t bytes_read = std::min(file_size, INIT_META_SIZE);
45
39
    std::vector<uint8_t> footer(bytes_read);
46
39
    RETURN_IF_ERROR(file->read_at(file_size - bytes_read, Slice(footer.data(), bytes_read),
47
39
                                  &bytes_read, io_ctx));
48
49
    // validate magic
50
39
    uint8_t* magic_ptr = footer.data() + bytes_read - 4;
51
39
    if (bytes_read < PARQUET_FOOTER_SIZE) {
52
0
        return Status::Corruption(
53
0
                "Read parquet file footer fail, bytes read: {}, file size: {}, path: {}",
54
0
                bytes_read, file_size, file->path().native());
55
39
    } else if (memcmp(magic_ptr, PARQUET_VERSION_NUMBER, sizeof(PARQUET_VERSION_NUMBER)) != 0) {
56
0
        return Status::Corruption(
57
0
                "Invalid magic number in parquet file, bytes read: {}, file size: {}, path: {}, "
58
0
                "read magic: {}",
59
0
                bytes_read, file_size, file->path().native(),
60
0
                std::string((char*)magic_ptr, sizeof(PARQUET_VERSION_NUMBER)));
61
0
    }
62
63
    // get metadata_size
64
39
    uint32_t metadata_size = decode_fixed32_le(footer.data() + bytes_read - PARQUET_FOOTER_SIZE);
65
39
    if (metadata_size > file_size - PARQUET_FOOTER_SIZE) {
66
0
        return Status::Corruption("Parquet footer size({}) is large than file size({})",
67
0
                                  metadata_size, file_size);
68
0
    }
69
39
    DorisUniqueBufferPtr<uint8_t> new_buff;
70
39
    uint8_t* meta_ptr;
71
39
    if (metadata_size > bytes_read - PARQUET_FOOTER_SIZE) {
72
0
        new_buff = make_unique_buffer<uint8_t>(metadata_size);
73
0
        RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size,
74
0
                                      Slice(new_buff.get(), metadata_size), &bytes_read, io_ctx));
75
0
        meta_ptr = new_buff.get();
76
39
    } else {
77
39
        meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size;
78
39
    }
79
80
39
    tparquet::FileMetaData t_metadata;
81
    // deserialize footer
82
39
    RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata));
83
39
    *file_metadata = std::make_unique<FileMetaData>(t_metadata, metadata_size);
84
39
    RETURN_IF_ERROR(
85
39
            (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz));
86
39
    *meta_size = PARQUET_FOOTER_SIZE + metadata_size;
87
39
    return Status::OK();
88
39
}
parquet_thrift_test.cpp:_ZN5dorisL19parse_thrift_footerESt10shared_ptrINS_2io10FileReaderEEPSt10unique_ptrINS_12FileMetaDataESt14default_deleteIS5_EEPmPNS1_9IOContextEbb
Line
Count
Source
42
4
                                  const bool enable_mapping_timestamp_tz) {
43
4
    size_t file_size = file->size();
44
4
    size_t bytes_read = std::min(file_size, INIT_META_SIZE);
45
4
    std::vector<uint8_t> footer(bytes_read);
46
4
    RETURN_IF_ERROR(file->read_at(file_size - bytes_read, Slice(footer.data(), bytes_read),
47
4
                                  &bytes_read, io_ctx));
48
49
    // validate magic
50
4
    uint8_t* magic_ptr = footer.data() + bytes_read - 4;
51
4
    if (bytes_read < PARQUET_FOOTER_SIZE) {
52
0
        return Status::Corruption(
53
0
                "Read parquet file footer fail, bytes read: {}, file size: {}, path: {}",
54
0
                bytes_read, file_size, file->path().native());
55
4
    } else if (memcmp(magic_ptr, PARQUET_VERSION_NUMBER, sizeof(PARQUET_VERSION_NUMBER)) != 0) {
56
0
        return Status::Corruption(
57
0
                "Invalid magic number in parquet file, bytes read: {}, file size: {}, path: {}, "
58
0
                "read magic: {}",
59
0
                bytes_read, file_size, file->path().native(),
60
0
                std::string((char*)magic_ptr, sizeof(PARQUET_VERSION_NUMBER)));
61
0
    }
62
63
    // get metadata_size
64
4
    uint32_t metadata_size = decode_fixed32_le(footer.data() + bytes_read - PARQUET_FOOTER_SIZE);
65
4
    if (metadata_size > file_size - PARQUET_FOOTER_SIZE) {
66
0
        return Status::Corruption("Parquet footer size({}) is large than file size({})",
67
0
                                  metadata_size, file_size);
68
0
    }
69
4
    DorisUniqueBufferPtr<uint8_t> new_buff;
70
4
    uint8_t* meta_ptr;
71
4
    if (metadata_size > bytes_read - PARQUET_FOOTER_SIZE) {
72
0
        new_buff = make_unique_buffer<uint8_t>(metadata_size);
73
0
        RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size,
74
0
                                      Slice(new_buff.get(), metadata_size), &bytes_read, io_ctx));
75
0
        meta_ptr = new_buff.get();
76
4
    } else {
77
4
        meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size;
78
4
    }
79
80
4
    tparquet::FileMetaData t_metadata;
81
    // deserialize footer
82
4
    RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata));
83
4
    *file_metadata = std::make_unique<FileMetaData>(t_metadata, metadata_size);
84
4
    RETURN_IF_ERROR(
85
4
            (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz));
86
4
    *meta_size = PARQUET_FOOTER_SIZE + metadata_size;
87
4
    return Status::OK();
88
4
}
vparquet_reader.cpp:_ZN5dorisL19parse_thrift_footerESt10shared_ptrINS_2io10FileReaderEEPSt10unique_ptrINS_12FileMetaDataESt14default_deleteIS5_EEPmPNS1_9IOContextEbb
Line
Count
Source
42
91
                                  const bool enable_mapping_timestamp_tz) {
43
91
    size_t file_size = file->size();
44
91
    size_t bytes_read = std::min(file_size, INIT_META_SIZE);
45
91
    std::vector<uint8_t> footer(bytes_read);
46
91
    RETURN_IF_ERROR(file->read_at(file_size - bytes_read, Slice(footer.data(), bytes_read),
47
91
                                  &bytes_read, io_ctx));
48
49
    // validate magic
50
91
    uint8_t* magic_ptr = footer.data() + bytes_read - 4;
51
91
    if (bytes_read < PARQUET_FOOTER_SIZE) {
52
0
        return Status::Corruption(
53
0
                "Read parquet file footer fail, bytes read: {}, file size: {}, path: {}",
54
0
                bytes_read, file_size, file->path().native());
55
91
    } else if (memcmp(magic_ptr, PARQUET_VERSION_NUMBER, sizeof(PARQUET_VERSION_NUMBER)) != 0) {
56
0
        return Status::Corruption(
57
0
                "Invalid magic number in parquet file, bytes read: {}, file size: {}, path: {}, "
58
0
                "read magic: {}",
59
0
                bytes_read, file_size, file->path().native(),
60
0
                std::string((char*)magic_ptr, sizeof(PARQUET_VERSION_NUMBER)));
61
0
    }
62
63
    // get metadata_size
64
91
    uint32_t metadata_size = decode_fixed32_le(footer.data() + bytes_read - PARQUET_FOOTER_SIZE);
65
91
    if (metadata_size > file_size - PARQUET_FOOTER_SIZE) {
66
0
        return Status::Corruption("Parquet footer size({}) is large than file size({})",
67
0
                                  metadata_size, file_size);
68
0
    }
69
91
    DorisUniqueBufferPtr<uint8_t> new_buff;
70
91
    uint8_t* meta_ptr;
71
91
    if (metadata_size > bytes_read - PARQUET_FOOTER_SIZE) {
72
0
        new_buff = make_unique_buffer<uint8_t>(metadata_size);
73
0
        RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size,
74
0
                                      Slice(new_buff.get(), metadata_size), &bytes_read, io_ctx));
75
0
        meta_ptr = new_buff.get();
76
91
    } else {
77
91
        meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size;
78
91
    }
79
80
91
    tparquet::FileMetaData t_metadata;
81
    // deserialize footer
82
91
    RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata));
83
91
    *file_metadata = std::make_unique<FileMetaData>(t_metadata, metadata_size);
84
91
    RETURN_IF_ERROR(
85
91
            (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz));
86
91
    *meta_size = PARQUET_FOOTER_SIZE + metadata_size;
87
91
    return Status::OK();
88
91
}
Unexecuted instantiation: parquet_metadata_reader.cpp:_ZN5dorisL19parse_thrift_footerESt10shared_ptrINS_2io10FileReaderEEPSt10unique_ptrINS_12FileMetaDataESt14default_deleteIS5_EEPmPNS1_9IOContextEbb
89
90
} // namespace doris