Coverage Report

Created: 2026-03-12 17:07

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/load_util.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/load_util.h"
19
20
#include <string>
21
22
#include "util/string_util.h"
23
24
namespace doris {
25
void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
26
                            TFileFormatType::type* format_type,
27
3.38k
                            TFileCompressType::type* compress_type) {
28
3.38k
    if (format_str.empty()) {
29
1.29k
        parse_format("CSV", compress_type_str, format_type, compress_type);
30
1.29k
        return;
31
1.29k
    }
32
33
    // Assign compress_type
34
2.09k
    if (iequal(compress_type_str, "GZ")) {
35
183
        *compress_type = TFileCompressType::GZ;
36
1.90k
    } else if (iequal(compress_type_str, "LZO")) {
37
2
        *compress_type = TFileCompressType::LZO;
38
1.90k
    } else if (iequal(compress_type_str, "BZ2")) {
39
8
        *compress_type = TFileCompressType::BZ2;
40
1.89k
    } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
41
10
        *compress_type = TFileCompressType::LZ4FRAME;
42
1.88k
    } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
43
1
        *compress_type = TFileCompressType::LZ4BLOCK;
44
1.88k
    } else if (iequal(compress_type_str, "LZOP")) {
45
5
        *compress_type = TFileCompressType::LZO;
46
1.88k
    } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
47
1
        *compress_type = TFileCompressType::SNAPPYBLOCK;
48
1.88k
    } else if (iequal(compress_type_str, "DEFLATE")) {
49
3
        *compress_type = TFileCompressType::DEFLATE;
50
1.87k
    } else {
51
1.87k
        *compress_type = TFileCompressType::PLAIN;
52
1.87k
    }
53
54
    // Assign format_type
55
2.09k
    *format_type = TFileFormatType::FORMAT_UNKNOWN;
56
2.09k
    if (iequal(format_str, "CSV")) {
57
1.65k
        if (compress_type_str.empty()) {
58
1.45k
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
59
1.45k
        } else if (iequal(compress_type_str, "GZ")) {
60
181
            *format_type = TFileFormatType::FORMAT_CSV_GZ;
61
181
        } else if (iequal(compress_type_str, "LZO")) {
62
2
            *format_type = TFileFormatType::FORMAT_CSV_LZO;
63
17
        } else if (iequal(compress_type_str, "BZ2")) {
64
6
            *format_type = TFileFormatType::FORMAT_CSV_BZ2;
65
11
        } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
66
8
            *format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
67
8
        } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
68
0
            *format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
69
3
        } else if (iequal(compress_type_str, "LZOP")) {
70
2
            *format_type = TFileFormatType::FORMAT_CSV_LZOP;
71
2
        } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
72
0
            *format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
73
1
        } else if (iequal(compress_type_str, "DEFLATE")) {
74
1
            *format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
75
1
        }
76
1.65k
    } else if (iequal(format_str, "HIVE_TEXT")) {
77
3
        *format_type = TFileFormatType::FORMAT_TEXT;
78
431
    } else if (iequal(format_str, "JSON")) {
79
400
        *format_type = TFileFormatType::FORMAT_JSON;
80
400
    } else if (iequal(format_str, "PARQUET")) {
81
12
        *format_type = TFileFormatType::FORMAT_PARQUET;
82
19
    } else if (iequal(format_str, "ORC")) {
83
8
        *format_type = TFileFormatType::FORMAT_ORC;
84
11
    } else if (iequal(format_str, "WAL")) {
85
0
        *format_type = TFileFormatType::FORMAT_WAL;
86
11
    } else if (iequal(format_str, "ARROW")) {
87
10
        *format_type = TFileFormatType::FORMAT_ARROW;
88
10
    }
89
2.09k
}
90
91
2.05k
bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
92
2.05k
    switch (format) {
93
1.42k
    case TFileFormatType::FORMAT_CSV_PLAIN:
94
1.43k
    case TFileFormatType::FORMAT_CSV_BZ2:
95
1.43k
    case TFileFormatType::FORMAT_CSV_DEFLATE:
96
1.61k
    case TFileFormatType::FORMAT_CSV_GZ:
97
1.62k
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
98
1.62k
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
99
1.62k
    case TFileFormatType::FORMAT_CSV_LZO:
100
1.62k
    case TFileFormatType::FORMAT_CSV_LZOP:
101
2.01k
    case TFileFormatType::FORMAT_JSON:
102
2.02k
    case TFileFormatType::FORMAT_TEXT:
103
2.02k
    case TFileFormatType::FORMAT_WAL:
104
2.03k
    case TFileFormatType::FORMAT_ARROW:
105
2.03k
        return true;
106
24
    default:
107
24
        return false;
108
2.05k
    }
109
0
    return false;
110
2.05k
}
111
} // namespace  doris