be/src/util/load_util.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "util/load_util.h"

#include <string>

#include "util/string_util.h"

namespace doris {
void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
                            TFileFormatType::type* format_type,
                            TFileCompressType::type* compress_type) {
    if (format_str.empty()) {
        parse_format("CSV", compress_type_str, format_type, compress_type);
        return;
    }

    // Assign compress_type
    if (iequal(compress_type_str, "GZ")) {
        *compress_type = TFileCompressType::GZ;
    } else if (iequal(compress_type_str, "LZO")) {
        *compress_type = TFileCompressType::LZO;
    } else if (iequal(compress_type_str, "BZ2")) {
        *compress_type = TFileCompressType::BZ2;
    } else if (iequal(compress_type_str, "ZSTD")) {
        *compress_type = TFileCompressType::ZSTD;
    } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
        *compress_type = TFileCompressType::LZ4FRAME;
    } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
        *compress_type = TFileCompressType::LZ4BLOCK;
    } else if (iequal(compress_type_str, "LZOP")) {
        *compress_type = TFileCompressType::LZO;
    } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
        *compress_type = TFileCompressType::SNAPPYBLOCK;
    } else if (iequal(compress_type_str, "DEFLATE")) {
        *compress_type = TFileCompressType::DEFLATE;
    } else {
        *compress_type = TFileCompressType::PLAIN;
    }

    // Assign format_type
    *format_type = TFileFormatType::FORMAT_UNKNOWN;
    if (iequal(format_str, "CSV")) {
        if (compress_type_str.empty()) {
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
        } else if (iequal(compress_type_str, "GZ")) {
            *format_type = TFileFormatType::FORMAT_CSV_GZ;
        } else if (iequal(compress_type_str, "LZO")) {
            *format_type = TFileFormatType::FORMAT_CSV_LZO;
        } else if (iequal(compress_type_str, "BZ2")) {
            *format_type = TFileFormatType::FORMAT_CSV_BZ2;
        } else if (iequal(compress_type_str, "ZSTD")) {
            *format_type = TFileFormatType::FORMAT_CSV_PLAIN;
        } else if (iequal(compress_type_str, "LZ4") || iequal(compress_type_str, "LZ4FRAME")) {
            *format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
        } else if (iequal(compress_type_str, "LZ4_BLOCK")) {
            *format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
        } else if (iequal(compress_type_str, "LZOP")) {
            *format_type = TFileFormatType::FORMAT_CSV_LZOP;
        } else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
            *format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
        } else if (iequal(compress_type_str, "DEFLATE")) {
            *format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
        }
    } else if (iequal(format_str, "HIVE_TEXT")) {
        *format_type = TFileFormatType::FORMAT_TEXT;
    } else if (iequal(format_str, "JSON")) {
        *format_type = TFileFormatType::FORMAT_JSON;
    } else if (iequal(format_str, "PARQUET")) {
        *format_type = TFileFormatType::FORMAT_PARQUET;
    } else if (iequal(format_str, "ORC")) {
        *format_type = TFileFormatType::FORMAT_ORC;
    } else if (iequal(format_str, "WAL")) {
        *format_type = TFileFormatType::FORMAT_WAL;
    } else if (iequal(format_str, "ARROW")) {
        *format_type = TFileFormatType::FORMAT_ARROW;
    }
}

bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
    switch (format) {
    case TFileFormatType::FORMAT_CSV_PLAIN:
    case TFileFormatType::FORMAT_CSV_BZ2:
    case TFileFormatType::FORMAT_CSV_DEFLATE:
    case TFileFormatType::FORMAT_CSV_GZ:
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
    case TFileFormatType::FORMAT_CSV_LZO:
    case TFileFormatType::FORMAT_CSV_LZOP:
    case TFileFormatType::FORMAT_JSON:
    case TFileFormatType::FORMAT_TEXT:
    case TFileFormatType::FORMAT_WAL:
    case TFileFormatType::FORMAT_ARROW:
        return true;
    default:
        return false;
    }
    return false;
}

bool LoadUtil::is_compressed_load(TFileCompressType::type compress_type,
                                  TFileFormatType::type format_type) {
    if (compress_type != TFileCompressType::UNKNOWN && compress_type != TFileCompressType::PLAIN) {
        return true;
    }

    switch (format_type) {
    case TFileFormatType::FORMAT_CSV_BZ2:
    case TFileFormatType::FORMAT_CSV_DEFLATE:
    case TFileFormatType::FORMAT_CSV_GZ:
    case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
    case TFileFormatType::FORMAT_CSV_LZ4FRAME:
    case TFileFormatType::FORMAT_CSV_LZO:
    case TFileFormatType::FORMAT_CSV_LZOP:
    case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK:
        return true;
    default:
        return false;
    }
}
} // namespace  doris

Line	Count	Source
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17
18		#include "util/load_util.h"
19
20		#include <string>
21
22		#include "util/string_util.h"
23
24		namespace doris {
25		void LoadUtil::parse_format(const std::string& format_str, const std::string& compress_type_str,
26		TFileFormatType::type* format_type,
27	6.89k	TFileCompressType::type* compress_type) {
28	6.89k	if (format_str.empty()) {
29	1.89k	parse_format("CSV", compress_type_str, format_type, compress_type);
30	1.89k	return;
31	1.89k	}
32
33		// Assign compress_type
34	5.00k	if (iequal(compress_type_str, "GZ")) {
35	157	*compress_type = TFileCompressType::GZ;
36	4.84k	} else if (iequal(compress_type_str, "LZO")) {
37	3	*compress_type = TFileCompressType::LZO;
38	4.84k	} else if (iequal(compress_type_str, "BZ2")) {
39	10	*compress_type = TFileCompressType::BZ2;
40	4.83k	} else if (iequal(compress_type_str, "ZSTD")) {
41	9	*compress_type = TFileCompressType::ZSTD;
42	4.82k	} else if (iequal(compress_type_str, "LZ4") \|\| iequal(compress_type_str, "LZ4FRAME")) {
43	12	*compress_type = TFileCompressType::LZ4FRAME;
44	4.81k	} else if (iequal(compress_type_str, "LZ4_BLOCK")) {
45	2	*compress_type = TFileCompressType::LZ4BLOCK;
46	4.81k	} else if (iequal(compress_type_str, "LZOP")) {
47	8	*compress_type = TFileCompressType::LZO;
48	4.80k	} else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
49	2	*compress_type = TFileCompressType::SNAPPYBLOCK;
50	4.80k	} else if (iequal(compress_type_str, "DEFLATE")) {
51	5	*compress_type = TFileCompressType::DEFLATE;
52	4.79k	} else {
53	4.79k	*compress_type = TFileCompressType::PLAIN;
54	4.79k	}
55
56		// Assign format_type
57	5.00k	*format_type = TFileFormatType::FORMAT_UNKNOWN;
58	5.00k	if (iequal(format_str, "CSV")) {
59	2.27k	if (compress_type_str.empty()) {
60	2.09k	*format_type = TFileFormatType::FORMAT_CSV_PLAIN;
61	2.09k	} else if (iequal(compress_type_str, "GZ")) {
62	154	*format_type = TFileFormatType::FORMAT_CSV_GZ;
63	154	} else if (iequal(compress_type_str, "LZO")) {
64	3	*format_type = TFileFormatType::FORMAT_CSV_LZO;
65	28	} else if (iequal(compress_type_str, "BZ2")) {
66	7	*format_type = TFileFormatType::FORMAT_CSV_BZ2;
67	21	} else if (iequal(compress_type_str, "ZSTD")) {
68	6	*format_type = TFileFormatType::FORMAT_CSV_PLAIN;
69	15	} else if (iequal(compress_type_str, "LZ4") \|\| iequal(compress_type_str, "LZ4FRAME")) {
70	9	*format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
71	9	} else if (iequal(compress_type_str, "LZ4_BLOCK")) {
72	0	*format_type = TFileFormatType::FORMAT_CSV_LZ4BLOCK;
73	6	} else if (iequal(compress_type_str, "LZOP")) {
74	3	*format_type = TFileFormatType::FORMAT_CSV_LZOP;
75	3	} else if (iequal(compress_type_str, "SNAPPY_BLOCK")) {
76	0	*format_type = TFileFormatType::FORMAT_CSV_SNAPPYBLOCK;
77	3	} else if (iequal(compress_type_str, "DEFLATE")) {
78	2	*format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
79	2	}
80	2.72k	} else if (iequal(format_str, "HIVE_TEXT")) {
81	3	*format_type = TFileFormatType::FORMAT_TEXT;
82	2.72k	} else if (iequal(format_str, "JSON")) {
83	2.68k	*format_type = TFileFormatType::FORMAT_JSON;
84	2.68k	} else if (iequal(format_str, "PARQUET")) {
85	15	*format_type = TFileFormatType::FORMAT_PARQUET;
86	18	} else if (iequal(format_str, "ORC")) {
87	11	*format_type = TFileFormatType::FORMAT_ORC;
88	11	} else if (iequal(format_str, "WAL")) {
89	0	*format_type = TFileFormatType::FORMAT_WAL;
90	10	} else if (iequal(format_str, "ARROW")) {
91	10	*format_type = TFileFormatType::FORMAT_ARROW;
92	10	}
93	5.00k	}
94
95	4.96k	bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
96	4.96k	switch (format) {
97	2.06k	case TFileFormatType::FORMAT_CSV_PLAIN:
98	2.07k	case TFileFormatType::FORMAT_CSV_BZ2:
99	2.07k	case TFileFormatType::FORMAT_CSV_DEFLATE:
100	2.23k	case TFileFormatType::FORMAT_CSV_GZ:
101	2.23k	case TFileFormatType::FORMAT_CSV_LZ4FRAME:
102	2.23k	case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
103	2.24k	case TFileFormatType::FORMAT_CSV_LZO:
104	2.24k	case TFileFormatType::FORMAT_CSV_LZOP:
105	4.91k	case TFileFormatType::FORMAT_JSON:
106	4.91k	case TFileFormatType::FORMAT_TEXT:
107	4.91k	case TFileFormatType::FORMAT_WAL:
108	4.92k	case TFileFormatType::FORMAT_ARROW:
109	4.92k	return true;
110	34	default:
111	34	return false;
112	4.96k	}
113	0	return false;
114	4.96k	}
115
116		bool LoadUtil::is_compressed_load(TFileCompressType::type compress_type,
117	444	TFileFormatType::type format_type) {
118	444	if (compress_type != TFileCompressType::UNKNOWN && compress_type != TFileCompressType::PLAIN) {
119	14	return true;
120	14	}
121
122	430	switch (format_type) {
123	0	case TFileFormatType::FORMAT_CSV_BZ2:
124	2	case TFileFormatType::FORMAT_CSV_DEFLATE:
125	4	case TFileFormatType::FORMAT_CSV_GZ:
126	4	case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
127	4	case TFileFormatType::FORMAT_CSV_LZ4FRAME:
128	4	case TFileFormatType::FORMAT_CSV_LZO:
129	4	case TFileFormatType::FORMAT_CSV_LZOP:
130	4	case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK:
131	4	return true;
132	426	default:
133	426	return false;
134	430	}
135	430	}
136		} // namespace doris

Coverage Report

Created: 2026-07-12 08:37