be/src/exec/scan/split_source_connector.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "exec/scan/split_source_connector.h" |
19 | | |
20 | | #include "runtime/exec_env.h" |
21 | | #include "runtime/query_context.h" |
22 | | |
23 | | namespace doris { |
24 | | |
25 | | using apache::thrift::transport::TTransportException; |
26 | | |
27 | 0 | Status LocalSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* range) { |
28 | 0 | std::lock_guard<std::mutex> l(_range_lock); |
29 | 0 | *has_next = false; |
30 | 0 | if (_scan_index < _scan_ranges.size()) { |
31 | 0 | auto& ranges = _scan_ranges[_scan_index].scan_range.ext_scan_range.file_scan_range.ranges; |
32 | 0 | if (_range_index < ranges.size()) { |
33 | 0 | *has_next = true; |
34 | 0 | *range = ranges[_range_index++]; |
35 | 0 | if (_range_index == ranges.size()) { |
36 | 0 | _scan_index++; |
37 | 0 | _range_index = 0; |
38 | 0 | } |
39 | 0 | } |
40 | 0 | } |
41 | 0 | return Status::OK(); |
42 | 0 | } |
43 | | |
44 | 0 | Status RemoteSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* range) { |
45 | 0 | std::lock_guard<std::mutex> l(_range_lock); |
46 | 0 | *has_next = false; |
47 | 0 | if (_scan_index == _scan_ranges.size() && !_last_batch) { |
48 | 0 | SCOPED_TIMER(_get_split_timer); |
49 | 0 | Status coord_status; |
50 | | // No need to set timeout because on FE side, there is a max fetch time |
51 | 0 | FrontendServiceConnection coord(_state->exec_env()->frontend_client_cache(), |
52 | 0 | _state->get_query_ctx()->coord_addr, &coord_status); |
53 | 0 | RETURN_IF_ERROR(coord_status); |
54 | 0 | TFetchSplitBatchRequest request; |
55 | 0 | request.__set_split_source_id(_split_source_id); |
56 | 0 | request.__set_max_num_splits(config::remote_split_source_batch_size); |
57 | 0 | TFetchSplitBatchResult result; |
58 | 0 | try { |
59 | 0 | coord->fetchSplitBatch(result, request); |
60 | 0 | if (result.__isset.status && result.status.status_code != TStatusCode::OK) { |
61 | 0 | return Status::IOError<false>("Failed to get batch of split source: {}", |
62 | 0 | result.status.error_msgs[0]); |
63 | 0 | } |
64 | 0 | } catch (std::exception& e) { |
65 | 0 | return Status::IOError<false>("Failed to get batch of split source: {}", e.what()); |
66 | 0 | } |
67 | 0 | _last_batch = result.splits.empty(); |
68 | 0 | _merge_ranges<TScanRangeLocations>(_scan_ranges, result.splits); |
69 | 0 | _scan_index = 0; |
70 | 0 | _range_index = 0; |
71 | 0 | } |
72 | 0 | if (_scan_index < _scan_ranges.size()) { |
73 | 0 | auto& ranges = _scan_ranges[_scan_index].scan_range.ext_scan_range.file_scan_range.ranges; |
74 | 0 | if (_range_index < ranges.size()) { |
75 | 0 | *has_next = true; |
76 | 0 | *range = ranges[_range_index++]; |
77 | 0 | if (_range_index == ranges.size()) { |
78 | 0 | _scan_index++; |
79 | 0 | _range_index = 0; |
80 | 0 | } |
81 | 0 | } |
82 | 0 | } |
83 | 0 | return Status::OK(); |
84 | 0 | } |
85 | | |
86 | | } // namespace doris |