Coverage Report

Created: 2025-06-13 16:38

/root/doris/be/src/util/load_util.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/load_util.h"
19
20
#include <string>
21
22
#include "util/string_util.h"
23
24
namespace doris {
25
void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
26
                            TFileFormatType::type* format_type,
27
20
                            TFileCompressType::type* compress_type) {
28
20
    if (format_str.empty()) {
29
1
        parse_format("CSV", compress_type_str, format_type, compress_type);
30
1
        return;
31
1
    }
32
33
    // Assign compress_type
34
19
    if (iequal(compress_type_str, "GZ")) {
35
2
        *compress_type = TFileCompressType::GZ;
36
17
    } else if (iequal(compress_type_str, "LZO")) {
37
1
        *compress_type = TFileCompressType::LZO;
38
16
    } else if (iequal(compress_type_str, "BZ2")) {
39
2
        *compress_type = TFileCompressType::BZ2;
40
14
    } else if (iequal(compress_type_str, "LZ4")) {
41
2
        *compress_type = TFileCompressType::LZ4FRAME;
42
12
    } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
43
1
        *compress_type = TFileCompressType::LZ4BLOCK;
44
11
    } else if (iequal(compress_type_str, "LZOP")) {
45
3
        *compress_type = TFileCompressType::LZO;
46
8
    } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
47
1
        *compress_type = TFileCompressType::SNAPPYBLOCK;
48
7
    } else if (iequal(compress_type_str, "DEFLATE")) {
49
2
        *compress_type = TFileCompressType::DEFLATE;
50
5
    } else {
51
5
        *compress_type = TFileCompressType::PLAIN;
52
5
    }
53
54
    // Assign format_type
55
19
    *format_type = TFileFormatType::FORMAT_UNKNOWN;
56
19
    if (iequal(format_str, "CSV")) {
57
7
        if (compress_type_str.empty()) {
58
1
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
59
6
        } else if (iequal(compress_type_str, "GZ")) {
60
1
            *format_type = TFileFormatType::FORMAT_CSV_GZ;
61
5
        } else if (iequal(compress_type_str, "LZO")) {
62
1
            *format_type = TFileFormatType::FORMAT_CSV_LZO;
63
4
        } else if (iequal(compress_type_str, "BZ2")) {
64
1
            *format_type = TFileFormatType::FORMAT_CSV_BZ2;
65
3
        } else if (iequal(compress_type_str, "LZ4")) {
66
1
            *format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
67
2
        } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
68
0
            *format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
69
2
        } else if (iequal(compress_type_str, "LZOP")) {
70
1
            *format_type = TFileFormatType::FORMAT_CSV_LZOP;
71
1
        } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
72
0
            *format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
73
1
        } else if (iequal(compress_type_str, "DEFLATE")) {
74
1
            *format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
75
1
        }
76
12
    } else if (iequal(format_str, "HIVE_TEXT")) {
77
0
        *format_type = TFileFormatType::FORMAT_TEXT;
78
12
    } else if (iequal(format_str, "JSON")) {
79
10
        *format_type = TFileFormatType::FORMAT_JSON;
80
10
    } else if (iequal(format_str, "PARQUET")) {
81
1
        *format_type = TFileFormatType::FORMAT_PARQUET;
82
1
    } else if (iequal(format_str, "ORC")) {
83
1
        *format_type = TFileFormatType::FORMAT_ORC;
84
1
    } else if (iequal(format_str, "WAL")) {
85
0
        *format_type = TFileFormatType::FORMAT_WAL;
86
0
    } else if (iequal(format_str, "ARROW")) {
87
0
        *format_type = TFileFormatType::FORMAT_ARROW;
88
0
    }
89
19
}
90
91
14
bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
92
14
    switch (format) {
93
1
    case TFileFormatType::FORMAT_CSV_PLAIN:
94
2
    case TFileFormatType::FORMAT_CSV_BZ2:
95
3
    case TFileFormatType::FORMAT_CSV_DEFLATE:
96
4
    case TFileFormatType::FORMAT_CSV_GZ:
97
5
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
98
5
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
99
6
    case TFileFormatType::FORMAT_CSV_LZO:
100
7
    case TFileFormatType::FORMAT_CSV_LZOP:
101
8
    case TFileFormatType::FORMAT_JSON:
102
8
    case TFileFormatType::FORMAT_TEXT:
103
8
    case TFileFormatType::FORMAT_WAL:
104
8
    case TFileFormatType::FORMAT_ARROW:
105
8
        return true;
106
6
    default:
107
6
        return false;
108
14
    }
109
0
    return false;
110
14
}
111
} // namespace  doris