Coverage Report

Created: 2026-06-25 00:28

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/util/load_util.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/load_util.h"
19
20
#include <string>
21
22
#include "util/string_util.h"
23
24
namespace doris {
25
void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
26
                            TFileFormatType::type* format_type,
27
24
                            TFileCompressType::type* compress_type) {
28
24
    if (format_str.empty()) {
29
2
        parse_format("CSV", compress_type_str, format_type, compress_type);
30
2
        return;
31
2
    }
32
33
    // Assign compress_type
34
22
    if (iequal(compress_type_str, "GZ")) {
35
2
        *compress_type = TFileCompressType::GZ;
36
20
    } else if (iequal(compress_type_str, "LZO")) {
37
1
        *compress_type = TFileCompressType::LZO;
38
19
    } else if (iequal(compress_type_str, "BZ2")) {
39
2
        *compress_type = TFileCompressType::BZ2;
40
17
    } else if (iequal(compress_type_str, "ZSTD")) {
41
3
        *compress_type = TFileCompressType::ZSTD;
42
14
    } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
43
2
        *compress_type = TFileCompressType::LZ4FRAME;
44
12
    } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
45
1
        *compress_type = TFileCompressType::LZ4BLOCK;
46
11
    } else if (iequal(compress_type_str, "LZOP")) {
47
3
        *compress_type = TFileCompressType::LZO;
48
8
    } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
49
1
        *compress_type = TFileCompressType::SNAPPYBLOCK;
50
7
    } else if (iequal(compress_type_str, "DEFLATE")) {
51
2
        *compress_type = TFileCompressType::DEFLATE;
52
5
    } else {
53
5
        *compress_type = TFileCompressType::PLAIN;
54
5
    }
55
56
    // Assign format_type
57
22
    *format_type = TFileFormatType::FORMAT_UNKNOWN;
58
22
    if (iequal(format_str, "CSV")) {
59
9
        if (compress_type_str.empty()) {
60
1
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
61
8
        } else if (iequal(compress_type_str, "GZ")) {
62
1
            *format_type = TFileFormatType::FORMAT_CSV_GZ;
63
7
        } else if (iequal(compress_type_str, "LZO")) {
64
1
            *format_type = TFileFormatType::FORMAT_CSV_LZO;
65
6
        } else if (iequal(compress_type_str, "BZ2")) {
66
1
            *format_type = TFileFormatType::FORMAT_CSV_BZ2;
67
5
        } else if (iequal(compress_type_str, "ZSTD")) {
68
2
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
69
3
        } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
70
1
            *format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
71
2
        } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
72
0
            *format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
73
2
        } else if (iequal(compress_type_str, "LZOP")) {
74
1
            *format_type = TFileFormatType::FORMAT_CSV_LZOP;
75
1
        } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
76
0
            *format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
77
1
        } else if (iequal(compress_type_str, "DEFLATE")) {
78
1
            *format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
79
1
        }
80
13
    } else if (iequal(format_str, "HIVE_TEXT")) {
81
0
        *format_type = TFileFormatType::FORMAT_TEXT;
82
13
    } else if (iequal(format_str, "JSON")) {
83
11
        *format_type = TFileFormatType::FORMAT_JSON;
84
11
    } else if (iequal(format_str, "PARQUET")) {
85
1
        *format_type = TFileFormatType::FORMAT_PARQUET;
86
1
    } else if (iequal(format_str, "ORC")) {
87
1
        *format_type = TFileFormatType::FORMAT_ORC;
88
1
    } else if (iequal(format_str, "WAL")) {
89
0
        *format_type = TFileFormatType::FORMAT_WAL;
90
0
    } else if (iequal(format_str, "ARROW")) {
91
0
        *format_type = TFileFormatType::FORMAT_ARROW;
92
0
    }
93
22
}
94
95
14
bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
96
14
    switch (format) {
97
1
    case TFileFormatType::FORMAT_CSV_PLAIN:
98
2
    case TFileFormatType::FORMAT_CSV_BZ2:
99
3
    case TFileFormatType::FORMAT_CSV_DEFLATE:
100
4
    case TFileFormatType::FORMAT_CSV_GZ:
101
5
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
102
5
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
103
6
    case TFileFormatType::FORMAT_CSV_LZO:
104
7
    case TFileFormatType::FORMAT_CSV_LZOP:
105
8
    case TFileFormatType::FORMAT_JSON:
106
8
    case TFileFormatType::FORMAT_TEXT:
107
8
    case TFileFormatType::FORMAT_WAL:
108
8
    case TFileFormatType::FORMAT_ARROW:
109
8
        return true;
110
6
    default:
111
6
        return false;
112
14
    }
113
0
    return false;
114
14
}
115
116
bool LoadUtil::is_compressed_load(TFileCompressType::type compress_type,
117
7
                                  TFileFormatType::type format_type) {
118
7
    if (compress_type != TFileCompressType::UNKNOWN && compress_type != TFileCompressType::PLAIN) {
119
2
        return true;
120
2
    }
121
122
5
    switch (format_type) {
123
0
    case TFileFormatType::FORMAT_CSV_BZ2:
124
1
    case TFileFormatType::FORMAT_CSV_DEFLATE:
125
2
    case TFileFormatType::FORMAT_CSV_GZ:
126
2
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
127
2
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
128
2
    case TFileFormatType::FORMAT_CSV_LZO:
129
2
    case TFileFormatType::FORMAT_CSV_LZOP:
130
2
    case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK:
131
2
        return true;
132
3
    default:
133
3
        return false;
134
5
    }
135
5
}
136
} // namespace  doris