Coverage Report

Created: 2026-08-18 15:11

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/format_v2/table/paimon_reader.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <functional>
21
#include <utility>
22
23
#include "format_v2/table_reader.h"
24
25
namespace doris {
26
struct DeleteFileDesc;
27
}
28
namespace doris::format::paimon {
29
30
class PaimonReader final : public format::TableReader {
31
public:
32
    ENABLE_FACTORY_CREATOR(PaimonReader);
33
15
    ~PaimonReader() final = default;
34
    Status prepare_split(const format::SplitReadOptions& options) override;
35
36
#ifdef BE_TEST
37
4
    void TEST_set_scan_params(TFileScanRangeParams* params) { _scan_params = params; }
38
4
    void TEST_set_projected_columns(std::vector<format::ColumnDefinition> columns) {
39
4
        _projected_columns = std::move(columns);
40
4
    }
41
4
    void TEST_set_format(format::FileFormat format) { _format = format; }
42
4
    format::TableColumnMappingMode TEST_mapping_mode() const { return mapping_mode(); }
43
7
    Status TEST_annotate_file_schema(std::vector<format::ColumnDefinition>* file_schema) {
44
7
        return annotate_file_schema(file_schema);
45
7
    }
46
4
    Status TEST_customize_file_scan_request(format::FileScanRequest* request) {
47
4
        return customize_file_scan_request(request);
48
4
    }
49
    Status TEST_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc,
50
4
                                           bool* has_delete_file) {
51
4
        return _parse_deletion_vector_file(t_desc, desc, has_delete_file);
52
4
    }
53
#endif
54
55
protected:
56
    format::TableColumnMappingMode mapping_mode() const override;
57
    Status annotate_file_schema(std::vector<format::ColumnDefinition>* file_schema) override;
58
    Status customize_file_scan_request(format::FileScanRequest* file_request) override;
59
60
    Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc,
61
                                       bool* has_delete_file) override;
62
63
private:
64
    int64_t _split_schema_id = -1;
65
    std::vector<format::LocalColumnIndex> _variant_schema_overrides;
66
};
67
68
// Paimon scans can contain both native data-file splits and serialized JNI splits in the same
69
// SplitSource. FileScannerV2 owns one table reader for the scanner lifetime, so this reader keeps
70
// native and JNI child readers internally and dispatches each split to the matching child reader.
71
class PaimonHybridReader final : public format::TableReader {
72
public:
73
7
    ~PaimonHybridReader() override = default;
74
75
    Status init(format::TableReadOptions&& options) override;
76
    Status prepare_split(const format::SplitReadOptions& options) override;
77
    Status refresh_conjuncts(VExprContextSPtrs conjuncts) override;
78
    Status get_block(Block* block, bool* eos) override;
79
    bool current_split_pruned() const override;
80
    bool current_split_uses_metadata_count() const override;
81
    Status abort_split() override;
82
    Status close() override;
83
    void set_batch_size(size_t batch_size) override;
84
    int64_t condition_cache_hit_count() const override;
85
86
#ifdef BE_TEST
87
3
    static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); }
88
    static Status TEST_to_file_format(const TFileRangeDesc& range,
89
5
                                      format::FileFormat* file_format) {
90
5
        return _to_file_format(range, file_format);
91
5
    }
92
2
    void TEST_install_batch_size_children() {
93
2
        _native_reader = std::make_unique<format::TableReader>();
94
2
        _jni_reader = std::make_unique<format::TableReader>();
95
2
    }
96
1
    std::pair<size_t, size_t> TEST_child_batch_sizes() const {
97
1
        return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()};
98
1
    }
99
1
    void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) {
100
1
        _native_reader->TEST_set_condition_cache_hit_count(native_hits);
101
1
        _jni_reader->TEST_set_condition_cache_hit_count(jni_hits);
102
1
    }
103
    void TEST_set_child_reader_factories(
104
            std::function<std::unique_ptr<format::TableReader>()> native_factory,
105
3
            std::function<std::unique_ptr<format::TableReader>()> jni_factory) {
106
3
        _test_native_reader_factory = std::move(native_factory);
107
3
        _test_jni_reader_factory = std::move(jni_factory);
108
3
    }
109
#endif
110
111
private:
112
    Status _ensure_current_split_reader(const format::SplitReadOptions& options);
113
    Status _init_child_reader(format::TableReader* reader, format::FileFormat file_format);
114
    Status _clone_conjuncts(VExprContextSPtrs* conjuncts) const;
115
    static bool _is_jni_split(const TFileRangeDesc& range);
116
    static Status _to_file_format(const TFileRangeDesc& range, format::FileFormat* file_format);
117
118
    std::unique_ptr<format::TableReader> _native_reader; // handle parquet/orc native splits
119
    std::unique_ptr<format::TableReader> _jni_reader;    // handle serialized JNI splits
120
    format::TableReader* _current_split_reader = nullptr;
121
#ifdef BE_TEST
122
    std::function<std::unique_ptr<format::TableReader>()> _test_native_reader_factory;
123
    std::function<std::unique_ptr<format::TableReader>()> _test_jni_reader_factory;
124
#endif
125
};
126
127
} // namespace doris::format::paimon