/root/doris/be/src/vec/runtime/partitioner.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "partitioner.h" |
19 | | |
20 | | #include "common/cast_set.h" |
21 | | #include "pipeline/local_exchange/local_exchange_sink_operator.h" |
22 | | #include "runtime/thread_context.h" |
23 | | #include "vec/columns/column_const.h" |
24 | | #include "vec/sink/vdata_stream_sender.h" |
25 | | |
26 | | namespace doris::vectorized { |
27 | | #include "common/compile_check_begin.h" |
28 | | |
29 | | template <typename ChannelIds> |
30 | | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block, |
31 | 27 | bool eos, bool* already_sent) const { |
32 | 27 | size_t rows = block->rows(); |
33 | | |
34 | 27 | if (rows > 0) { |
35 | 27 | auto column_to_keep = block->columns(); |
36 | | |
37 | 27 | int result_size = cast_set<int>(_partition_expr_ctxs.size()); |
38 | 27 | std::vector<int> result(result_size); |
39 | | |
40 | 27 | _hash_vals.resize(rows); |
41 | 27 | std::fill(_hash_vals.begin(), _hash_vals.end(), 0); |
42 | 27 | auto* __restrict hashes = _hash_vals.data(); |
43 | 27 | { RETURN_IF_ERROR(_get_partition_column_result(block, result)); } |
44 | 54 | for (int j = 0; j < result_size; ++j) { |
45 | 27 | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); |
46 | 27 | if (is_const) { |
47 | 0 | continue; |
48 | 0 | } |
49 | 27 | _do_hash(col, hashes, j); |
50 | 27 | } |
51 | | |
52 | 3.14M | for (size_t i = 0; i < rows; i++) { |
53 | 3.14M | hashes[i] = ChannelIds()(hashes[i], _partition_count); |
54 | 3.14M | } |
55 | | |
56 | 27 | { Block::erase_useless_column(block, column_to_keep); } |
57 | 27 | } |
58 | 27 | return Status::OK(); |
59 | 27 | } _ZNK5doris10vectorized20Crc32HashPartitionerINS0_17ShuffleChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS0_5BlockEbPb Line | Count | Source | 31 | 21 | bool eos, bool* already_sent) const { | 32 | 21 | size_t rows = block->rows(); | 33 | | | 34 | 21 | if (rows > 0) { | 35 | 21 | auto column_to_keep = block->columns(); | 36 | | | 37 | 21 | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 38 | 21 | std::vector<int> result(result_size); | 39 | | | 40 | 21 | _hash_vals.resize(rows); | 41 | 21 | std::fill(_hash_vals.begin(), _hash_vals.end(), 0); | 42 | 21 | auto* __restrict hashes = _hash_vals.data(); | 43 | 21 | { RETURN_IF_ERROR(_get_partition_column_result(block, result)); } | 44 | 42 | for (int j = 0; j < result_size; ++j) { | 45 | 21 | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 46 | 21 | if (is_const) { | 47 | 0 | continue; | 48 | 0 | } | 49 | 21 | _do_hash(col, hashes, j); | 50 | 21 | } | 51 | | | 52 | 231 | for (size_t i = 0; i < rows; i++) { | 53 | 210 | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 54 | 210 | } | 55 | | | 56 | 21 | { Block::erase_useless_column(block, column_to_keep); } | 57 | 21 | } | 58 | 21 | return Status::OK(); | 59 | 21 | } |
_ZNK5doris10vectorized20Crc32HashPartitionerINS0_24SpillPartitionChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS0_5BlockEbPb Line | Count | Source | 31 | 6 | bool eos, bool* already_sent) const { | 32 | 6 | size_t rows = block->rows(); | 33 | | | 34 | 6 | if (rows > 0) { | 35 | 6 | auto column_to_keep = block->columns(); | 36 | | | 37 | 6 | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 38 | 6 | std::vector<int> result(result_size); | 39 | | | 40 | 6 | _hash_vals.resize(rows); | 41 | 6 | std::fill(_hash_vals.begin(), _hash_vals.end(), 0); | 42 | 6 | auto* __restrict hashes = _hash_vals.data(); | 43 | 6 | { RETURN_IF_ERROR(_get_partition_column_result(block, result)); } | 44 | 12 | for (int j = 0; j < result_size; ++j) { | 45 | 6 | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 46 | 6 | if (is_const) { | 47 | 0 | continue; | 48 | 0 | } | 49 | 6 | _do_hash(col, hashes, j); | 50 | 6 | } | 51 | | | 52 | 3.14M | for (size_t i = 0; i < rows; i++) { | 53 | 3.14M | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 54 | 3.14M | } | 55 | | | 56 | 6 | { Block::erase_useless_column(block, column_to_keep); } | 57 | 6 | } | 58 | 6 | return Status::OK(); | 59 | 6 | } |
|
60 | | |
61 | | template <typename ChannelIds> |
62 | | void Crc32HashPartitioner<ChannelIds>::_do_hash(const ColumnPtr& column, |
63 | 27 | uint32_t* __restrict result, int idx) const { |
64 | 27 | column->update_crcs_with_value(result, _partition_expr_ctxs[idx]->root()->type().type, |
65 | 27 | cast_set<uint32_t>(column->size())); |
66 | 27 | } _ZNK5doris10vectorized20Crc32HashPartitionerINS0_17ShuffleChannelIdsEE8_do_hashERKNS_3COWINS0_7IColumnEE13immutable_ptrIS5_EEPji Line | Count | Source | 63 | 21 | uint32_t* __restrict result, int idx) const { | 64 | 21 | column->update_crcs_with_value(result, _partition_expr_ctxs[idx]->root()->type().type, | 65 | 21 | cast_set<uint32_t>(column->size())); | 66 | 21 | } |
_ZNK5doris10vectorized20Crc32HashPartitionerINS0_24SpillPartitionChannelIdsEE8_do_hashERKNS_3COWINS0_7IColumnEE13immutable_ptrIS5_EEPji Line | Count | Source | 63 | 6 | uint32_t* __restrict result, int idx) const { | 64 | 6 | column->update_crcs_with_value(result, _partition_expr_ctxs[idx]->root()->type().type, | 65 | 6 | cast_set<uint32_t>(column->size())); | 66 | 6 | } |
|
67 | | |
68 | | template <typename ChannelIds> |
69 | | Status Crc32HashPartitioner<ChannelIds>::clone(RuntimeState* state, |
70 | 3 | std::unique_ptr<PartitionerBase>& partitioner) { |
71 | 3 | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(cast_set<int>(_partition_count)); |
72 | | |
73 | 3 | partitioner.reset(new_partitioner); |
74 | 3 | new_partitioner->_partition_expr_ctxs.resize(_partition_expr_ctxs.size()); |
75 | 6 | for (size_t i = 0; i < _partition_expr_ctxs.size(); i++) { |
76 | 3 | RETURN_IF_ERROR( |
77 | 3 | _partition_expr_ctxs[i]->clone(state, new_partitioner->_partition_expr_ctxs[i])); |
78 | 3 | } |
79 | 3 | return Status::OK(); |
80 | 3 | } Unexecuted instantiation: _ZN5doris10vectorized20Crc32HashPartitionerINS0_17ShuffleChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS0_15PartitionerBaseESt14default_deleteIS7_EE _ZN5doris10vectorized20Crc32HashPartitionerINS0_24SpillPartitionChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS0_15PartitionerBaseESt14default_deleteIS7_EE Line | Count | Source | 70 | 3 | std::unique_ptr<PartitionerBase>& partitioner) { | 71 | 3 | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(cast_set<int>(_partition_count)); | 72 | | | 73 | 3 | partitioner.reset(new_partitioner); | 74 | 3 | new_partitioner->_partition_expr_ctxs.resize(_partition_expr_ctxs.size()); | 75 | 6 | for (size_t i = 0; i < _partition_expr_ctxs.size(); i++) { | 76 | 3 | RETURN_IF_ERROR( | 77 | 3 | _partition_expr_ctxs[i]->clone(state, new_partitioner->_partition_expr_ctxs[i])); | 78 | 3 | } | 79 | 3 | return Status::OK(); | 80 | 3 | } |
|
81 | | |
82 | | template class Crc32HashPartitioner<ShuffleChannelIds>; |
83 | | template class Crc32HashPartitioner<SpillPartitionChannelIds>; |
84 | | |
85 | | } // namespace doris::vectorized |