Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <cstdint> |
21 | | #include <map> |
22 | | #include <memory> |
23 | | #include <mutex> |
24 | | #include <unordered_map> |
25 | | #include <vector> |
26 | | |
27 | | #include "common/status.h" |
28 | | #include "core/block/block.h" |
29 | | #include "exec/common/hash_table/phmap_fwd_decl.h" |
30 | | #include "storage/tablet_info.h" |
31 | | #include "util/bitmap.h" |
32 | | #include "util/uid_util.h" |
33 | | |
34 | | namespace doris { |
35 | | |
36 | | class AdaptiveRandomBucketState { |
37 | | public: |
38 | 2.66k | explicit AdaptiveRandomBucketState(UniqueId load_id) : _load_id(load_id) {} |
39 | | |
40 | | void init_partition(int32_t sender_id, int64_t partition_id, |
41 | | const std::vector<int64_t>& tablets, |
42 | | const std::vector<int32_t>& bucket_seqs, int32_t start_tablet_idx); |
43 | | int64_t current_tablet(int32_t sender_id, int64_t partition_id); |
44 | | void rotate_by_tablet(int32_t sender_id, int64_t partition_id, int64_t tablet_id); |
45 | | |
46 | | private: |
47 | | struct PartitionState { |
48 | | int64_t partition_id = -1; |
49 | | std::vector<int64_t> tablets; |
50 | | std::vector<int32_t> bucket_seqs; |
51 | | int32_t tablet_pos = 0; |
52 | | int64_t current_tablet_id = -1; |
53 | | }; |
54 | | |
55 | | std::mutex _mutex; |
56 | | UniqueId _load_id; |
57 | | std::unordered_map<int32_t, std::unordered_map<int64_t, PartitionState>> |
58 | | _sender_partition_states; |
59 | | }; |
60 | | |
61 | | class OlapTabletFinder { |
62 | | public: |
63 | | // FIND_TABLET_EVERY_ROW is used for hash distribution info, which indicates that we |
64 | | // should compute tablet index for every row |
65 | | // FIND_TABLET_EVERY_BATCH is used for random distribution info, which indicates that we should |
66 | | // compute tablet index for every row batch |
67 | | // FIND_TABLET_EVERY_SINK is used for random distribution info when load_to_single_tablet set to true, |
68 | | // which indicates that we should only compute tablet index in the corresponding partition once for the |
69 | | // whole time in olap table sink |
70 | | // FIND_TABLET_RANDOM_BUCKET is used for V1 receiver-side random bucket mode. |
71 | | enum FindTabletMode { |
72 | | FIND_TABLET_EVERY_ROW, |
73 | | FIND_TABLET_EVERY_BATCH, |
74 | | FIND_TABLET_EVERY_SINK, |
75 | | FIND_TABLET_RANDOM_BUCKET |
76 | | }; |
77 | | |
78 | | OlapTabletFinder(VOlapTablePartitionParam* vpartition, FindTabletMode mode) |
79 | 42.5k | : _vpartition(vpartition), _find_tablet_mode(mode), _filter_bitmap(1024) {}; |
80 | | |
81 | | Status find_tablets(RuntimeState* state, Block* block, int rows, |
82 | | std::vector<VOlapTablePartition*>& partitions, |
83 | | std::vector<uint32_t>& tablet_index, std::vector<bool>& skip, |
84 | | std::vector<uint32_t>* miss_rows = nullptr); |
85 | | |
86 | 186 | bool is_find_tablet_every_sink() { |
87 | 186 | return _find_tablet_mode == FindTabletMode::FIND_TABLET_EVERY_SINK; |
88 | 186 | } |
89 | | |
90 | 107M | bool is_adaptive_random_bucket() const { |
91 | 107M | return _find_tablet_mode == FindTabletMode::FIND_TABLET_RANDOM_BUCKET; |
92 | 107M | } |
93 | | |
94 | 0 | bool is_single_tablet() { |
95 | 0 | return _find_tablet_mode != FindTabletMode::FIND_TABLET_RANDOM_BUCKET && |
96 | 0 | _partition_to_tablet_map.size() == 1; |
97 | 0 | } |
98 | | |
99 | | // all partitions for multi find-processes of its relative writer. |
100 | 88.1k | const flat_hash_set<int64_t>& partition_ids() { return _partition_ids; } |
101 | | |
102 | 85.5k | int64_t num_filtered_rows() const { return _num_filtered_rows; } |
103 | | |
104 | 42.7k | int64_t num_immutable_partition_filtered_rows() const { |
105 | 42.7k | return _num_immutable_partition_filtered_rows; |
106 | 42.7k | } |
107 | | |
108 | 37.7k | Bitmap& filter_bitmap() { return _filter_bitmap; } |
109 | | |
110 | | private: |
111 | | VOlapTablePartitionParam* _vpartition = nullptr; |
112 | | FindTabletMode _find_tablet_mode; |
113 | | std::map<VOlapTablePartition*, int64_t> _partition_to_tablet_map; |
114 | | flat_hash_set<int64_t> _partition_ids; |
115 | | |
116 | | int64_t _num_filtered_rows = 0; |
117 | | int64_t _num_immutable_partition_filtered_rows = 0; |
118 | | Bitmap _filter_bitmap; |
119 | | }; |
120 | | |
121 | | } // namespace doris |