be/src/exec/scan/file_scan_range_utils.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include "common/status.h" |
21 | | #include "gen_cpp/PlanNodes_types.h" |
22 | | |
23 | | namespace doris { |
24 | | |
25 | | // Return only static classifications: malformed wire payloads must not enter error messages. |
26 | 16 | inline const char* file_encryption_validation_error(const TFileEncryptionInfo& info) { |
27 | 16 | if (!info.__isset.version) { |
28 | 1 | return "File encryption [INVALID_DESCRIPTOR]"; |
29 | 1 | } |
30 | 15 | if (info.version != 1) { |
31 | 1 | return "File encryption [UNSUPPORTED_VERSION]"; |
32 | 1 | } |
33 | 14 | const auto length = info.key.size(); |
34 | 14 | if (!info.__isset.key || (length != 16 && length != 24 && length != 32) || |
35 | 14 | !info.__isset.aad_prefix) { |
36 | 2 | return "File encryption [INVALID_DESCRIPTOR]"; |
37 | 2 | } |
38 | 12 | return nullptr; |
39 | 14 | } |
40 | | |
41 | | inline Status validate_file_scan_support(const TFileScanRangeParams& params, |
42 | 23 | const TFileRangeDesc* range = nullptr) { |
43 | | // Deprecated table markers must never fall through to plain file reads: that would lose |
44 | | // Hive transaction visibility and LakeSoul merge semantics, including in metadata counts. |
45 | 23 | auto validate_table = [](const TTableFormatFileDesc& table) -> Status { |
46 | 16 | if (table.table_format_type == "transactional_hive") { |
47 | 3 | return Status::NotSupported("Hive transactional table reads are no longer supported"); |
48 | 3 | } |
49 | 13 | if (table.table_format_type == "lakesoul") { |
50 | 3 | return Status::NotSupported("LakeSoul reads are no longer supported"); |
51 | 3 | } |
52 | 10 | if (table.__isset.iceberg_params) { |
53 | 8 | const auto& iceberg = table.iceberg_params; |
54 | 8 | bool encrypted = iceberg.__isset.encryption_info; |
55 | 8 | if (encrypted) { |
56 | 2 | if (const auto* error = file_encryption_validation_error(iceberg.encryption_info)) { |
57 | 0 | return Status::InvalidArgument(error); |
58 | 0 | } |
59 | 2 | } |
60 | 8 | if (iceberg.__isset.delete_files) { |
61 | 6 | for (const auto& file : iceberg.delete_files) { |
62 | 6 | if (file.__isset.encryption_info) { |
63 | 6 | if (const auto* error = |
64 | 6 | file_encryption_validation_error(file.encryption_info)) { |
65 | 0 | return Status::InvalidArgument(error); |
66 | 0 | } |
67 | 6 | encrypted = true; |
68 | 6 | } |
69 | 6 | } |
70 | 6 | } |
71 | | // A plain data file can still have encrypted deletes. Reject before reader selection, |
72 | | // caches or metadata COUNT shortcuts until native decryption is implemented. |
73 | 8 | if (encrypted) { |
74 | 8 | return Status::NotSupported("File encryption [READ_UNAVAILABLE]"); |
75 | 8 | } |
76 | 8 | } |
77 | 2 | return Status::OK(); |
78 | 10 | }; |
79 | 23 | if (params.__isset.table_format_params) { |
80 | 10 | RETURN_IF_ERROR(validate_table(params.table_format_params)); |
81 | 10 | } |
82 | 15 | if (range != nullptr && range->__isset.table_format_params) { |
83 | 6 | RETURN_IF_ERROR(validate_table(range->table_format_params)); |
84 | 6 | } |
85 | 9 | const auto format = range != nullptr && range->__isset.format_type ? range->format_type |
86 | 9 | : params.format_type; |
87 | 9 | if (format == TFileFormatType::FORMAT_AVRO) { |
88 | 3 | return Status::NotSupported("Avro file reads are no longer supported"); |
89 | 3 | } |
90 | 6 | return Status::OK(); |
91 | 9 | } |
92 | | |
93 | 20 | inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) { |
94 | | // Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a |
95 | | // missing data or delete file would silently return an incomplete or incorrect snapshot. |
96 | 20 | return ignore_not_found && !(range.__isset.table_format_params && |
97 | 13 | range.table_format_params.table_format_type == "iceberg"); |
98 | 20 | } |
99 | | |
100 | | } // namespace doris |