Coverage Report

Created: 2024-11-21 21:13

/root/doris/be/src/util/load_util.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/load_util.h"
19
20
#include <string>
21
22
#include "util/string_util.h"
23
24
namespace doris {
25
void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
26
                            TFileFormatType::type* format_type,
27
12
                            TFileCompressType::type* compress_type) {
28
12
    if (format_str.empty()) {
29
1
        parse_format("CSV", compress_type_str, format_type, compress_type);
30
1
        return;
31
1
    }
32
11
    *compress_type = TFileCompressType::PLAIN;
33
11
    *format_type = TFileFormatType::FORMAT_UNKNOWN;
34
11
    if (iequal(format_str, "CSV")) {
35
7
        if (compress_type_str.empty()) {
36
1
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
37
6
        } else if (iequal(compress_type_str, "GZ")) {
38
1
            *format_type = TFileFormatType::FORMAT_CSV_GZ;
39
1
            *compress_type = TFileCompressType::GZ;
40
5
        } else if (iequal(compress_type_str, "LZO")) {
41
1
            *format_type = TFileFormatType::FORMAT_CSV_LZO;
42
1
            *compress_type = TFileCompressType::LZO;
43
4
        } else if (iequal(compress_type_str, "BZ2")) {
44
1
            *format_type = TFileFormatType::FORMAT_CSV_BZ2;
45
1
            *compress_type = TFileCompressType::BZ2;
46
3
        } else if (iequal(compress_type_str, "LZ4")) {
47
1
            *format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
48
1
            *compress_type = TFileCompressType::LZ4FRAME;
49
2
        } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
50
0
            *format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
51
0
            *compress_type = TFileCompressType::LZ4BLOCK;
52
2
        } else if (iequal(compress_type_str, "LZOP")) {
53
1
            *format_type = TFileFormatType::FORMAT_CSV_LZOP;
54
1
            *compress_type = TFileCompressType::LZO;
55
1
        } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
56
0
            *format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
57
0
            *compress_type = TFileCompressType::SNAPPYBLOCK;
58
1
        } else if (iequal(compress_type_str, "DEFLATE")) {
59
1
            *format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
60
1
            *compress_type = TFileCompressType::DEFLATE;
61
1
        }
62
7
    } else if (iequal(format_str, "JSON")) {
63
2
        if (compress_type_str.empty()) {
64
1
            *format_type = TFileFormatType::FORMAT_JSON;
65
1
        }
66
2
    } else if (iequal(format_str, "PARQUET")) {
67
1
        *format_type = TFileFormatType::FORMAT_PARQUET;
68
1
    } else if (iequal(format_str, "ORC")) {
69
1
        *format_type = TFileFormatType::FORMAT_ORC;
70
1
    } else if (iequal(format_str, "WAL")) {
71
0
        *format_type = TFileFormatType::FORMAT_WAL;
72
0
    } else if (iequal(format_str, "ARROW")) {
73
0
        *format_type = TFileFormatType::FORMAT_ARROW;
74
0
    }
75
11
    return;
76
12
}
77
78
14
bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
79
14
    switch (format) {
80
1
    case TFileFormatType::FORMAT_CSV_PLAIN:
81
2
    case TFileFormatType::FORMAT_CSV_BZ2:
82
3
    case TFileFormatType::FORMAT_CSV_DEFLATE:
83
4
    case TFileFormatType::FORMAT_CSV_GZ:
84
5
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
85
5
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
86
6
    case TFileFormatType::FORMAT_CSV_LZO:
87
7
    case TFileFormatType::FORMAT_CSV_LZOP:
88
8
    case TFileFormatType::FORMAT_JSON:
89
8
    case TFileFormatType::FORMAT_WAL:
90
8
    case TFileFormatType::FORMAT_ARROW:
91
8
        return true;
92
6
    default:
93
6
        return false;
94
14
    }
95
0
    return false;
96
14
}
97
} // namespace  doris