be/src/exec/partitioner/partitioner.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "exec/partitioner/partitioner.h" |
19 | | |
20 | | #include "common/cast_set.h" |
21 | | #include "common/status.h" |
22 | | #include "core/column/column_const.h" |
23 | | #include "exec/exchange/local_exchange_sink_operator.h" |
24 | | #include "exec/exchange/vdata_stream_sender.h" |
25 | | #include "runtime/thread_context.h" |
26 | | |
27 | | namespace doris { |
28 | | |
29 | | template <typename ChannelIds> |
30 | 286k | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block) const { |
31 | 286k | size_t rows = block->rows(); |
32 | | |
33 | 286k | if (rows > 0) { |
34 | 49.8k | auto column_to_keep = block->columns(); |
35 | | |
36 | 49.8k | int result_size = cast_set<int>(_partition_expr_ctxs.size()); |
37 | 49.8k | std::vector<int> result(result_size); |
38 | | |
39 | 49.8k | _initialize_hash_vals(rows); |
40 | 49.8k | auto* __restrict hashes = _hash_vals.data(); |
41 | 49.8k | RETURN_IF_ERROR(_get_partition_column_result(block, result)); |
42 | 124k | for (int j = 0; j < result_size; ++j) { |
43 | 74.4k | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); |
44 | 74.4k | if (is_const) { |
45 | 2 | continue; |
46 | 2 | } |
47 | 74.4k | _do_hash(col, hashes, j); |
48 | 74.4k | } |
49 | | |
50 | 32.9M | for (size_t i = 0; i < rows; i++) { |
51 | 32.8M | hashes[i] = ChannelIds()(hashes[i], _partition_count); |
52 | 32.8M | } |
53 | | |
54 | 49.8k | Block::erase_useless_column(block, column_to_keep); |
55 | 49.8k | } |
56 | 286k | return Status::OK(); |
57 | 286k | } _ZNK5doris20Crc32HashPartitionerINS_17ShuffleChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS_5BlockE Line | Count | Source | 30 | 11.1k | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block) const { | 31 | 11.1k | size_t rows = block->rows(); | 32 | | | 33 | 11.1k | if (rows > 0) { | 34 | 4.90k | auto column_to_keep = block->columns(); | 35 | | | 36 | 4.90k | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 37 | 4.90k | std::vector<int> result(result_size); | 38 | | | 39 | 4.90k | _initialize_hash_vals(rows); | 40 | 4.90k | auto* __restrict hashes = _hash_vals.data(); | 41 | 4.90k | RETURN_IF_ERROR(_get_partition_column_result(block, result)); | 42 | 10.6k | for (int j = 0; j < result_size; ++j) { | 43 | 5.78k | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 44 | 5.78k | if (is_const) { | 45 | 0 | continue; | 46 | 0 | } | 47 | 5.78k | _do_hash(col, hashes, j); | 48 | 5.78k | } | 49 | | | 50 | 2.50M | for (size_t i = 0; i < rows; i++) { | 51 | 2.49M | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 52 | 2.49M | } | 53 | | | 54 | 4.90k | Block::erase_useless_column(block, column_to_keep); | 55 | 4.90k | } | 56 | 11.1k | return Status::OK(); | 57 | 11.1k | } |
_ZNK5doris20Crc32HashPartitionerINS_24SpillPartitionChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS_5BlockE Line | Count | Source | 30 | 8 | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block) const { | 31 | 8 | size_t rows = block->rows(); | 32 | | | 33 | 8 | if (rows > 0) { | 34 | 8 | auto column_to_keep = block->columns(); | 35 | | | 36 | 8 | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 37 | 8 | std::vector<int> result(result_size); | 38 | | | 39 | 8 | _initialize_hash_vals(rows); | 40 | 8 | auto* __restrict hashes = _hash_vals.data(); | 41 | 8 | RETURN_IF_ERROR(_get_partition_column_result(block, result)); | 42 | 16 | for (int j = 0; j < result_size; ++j) { | 43 | 8 | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 44 | 8 | if (is_const) { | 45 | 0 | continue; | 46 | 0 | } | 47 | 8 | _do_hash(col, hashes, j); | 48 | 8 | } | 49 | | | 50 | 3.14M | for (size_t i = 0; i < rows; i++) { | 51 | 3.14M | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 52 | 3.14M | } | 53 | | | 54 | 8 | Block::erase_useless_column(block, column_to_keep); | 55 | 8 | } | 56 | 8 | return Status::OK(); | 57 | 8 | } |
_ZNK5doris20Crc32HashPartitionerINS_26SpillRePartitionChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS_5BlockE Line | Count | Source | 30 | 8 | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block) const { | 31 | 8 | size_t rows = block->rows(); | 32 | | | 33 | 8 | if (rows > 0) { | 34 | 8 | auto column_to_keep = block->columns(); | 35 | | | 36 | 8 | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 37 | 8 | std::vector<int> result(result_size); | 38 | | | 39 | 8 | _initialize_hash_vals(rows); | 40 | 8 | auto* __restrict hashes = _hash_vals.data(); | 41 | 8 | RETURN_IF_ERROR(_get_partition_column_result(block, result)); | 42 | 16 | for (int j = 0; j < result_size; ++j) { | 43 | 8 | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 44 | 8 | if (is_const) { | 45 | 0 | continue; | 46 | 0 | } | 47 | 8 | _do_hash(col, hashes, j); | 48 | 8 | } | 49 | | | 50 | 28 | for (size_t i = 0; i < rows; i++) { | 51 | 20 | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 52 | 20 | } | 53 | | | 54 | 8 | Block::erase_useless_column(block, column_to_keep); | 55 | 8 | } | 56 | 8 | return Status::OK(); | 57 | 8 | } |
_ZNK5doris20Crc32HashPartitionerINS_15ShiftChannelIdsEE15do_partitioningEPNS_12RuntimeStateEPNS_5BlockE Line | Count | Source | 30 | 275k | Status Crc32HashPartitioner<ChannelIds>::do_partitioning(RuntimeState* state, Block* block) const { | 31 | 275k | size_t rows = block->rows(); | 32 | | | 33 | 275k | if (rows > 0) { | 34 | 44.9k | auto column_to_keep = block->columns(); | 35 | | | 36 | 44.9k | int result_size = cast_set<int>(_partition_expr_ctxs.size()); | 37 | 44.9k | std::vector<int> result(result_size); | 38 | | | 39 | 44.9k | _initialize_hash_vals(rows); | 40 | 44.9k | auto* __restrict hashes = _hash_vals.data(); | 41 | 44.9k | RETURN_IF_ERROR(_get_partition_column_result(block, result)); | 42 | 113k | for (int j = 0; j < result_size; ++j) { | 43 | 68.6k | const auto& [col, is_const] = unpack_if_const(block->get_by_position(result[j]).column); | 44 | 68.6k | if (is_const) { | 45 | 2 | continue; | 46 | 2 | } | 47 | 68.6k | _do_hash(col, hashes, j); | 48 | 68.6k | } | 49 | | | 50 | 27.2M | for (size_t i = 0; i < rows; i++) { | 51 | 27.2M | hashes[i] = ChannelIds()(hashes[i], _partition_count); | 52 | 27.2M | } | 53 | | | 54 | 44.9k | Block::erase_useless_column(block, column_to_keep); | 55 | 44.9k | } | 56 | 275k | return Status::OK(); | 57 | 275k | } |
|
58 | | |
59 | | template <typename ChannelIds> |
60 | | void Crc32HashPartitioner<ChannelIds>::_do_hash(const ColumnPtr& column, |
61 | 5.80k | HashValType* __restrict result, int idx) const { |
62 | 5.80k | column->update_crcs_with_value( |
63 | 5.80k | result, _partition_expr_ctxs[idx]->root()->data_type()->get_primitive_type(), |
64 | 5.80k | cast_set<HashValType>(column->size())); |
65 | 5.80k | } _ZNK5doris20Crc32HashPartitionerINS_17ShuffleChannelIdsEE8_do_hashERKNS_3COWINS_7IColumnEE13immutable_ptrIS4_EEPji Line | Count | Source | 61 | 5.78k | HashValType* __restrict result, int idx) const { | 62 | 5.78k | column->update_crcs_with_value( | 63 | 5.78k | result, _partition_expr_ctxs[idx]->root()->data_type()->get_primitive_type(), | 64 | 5.78k | cast_set<HashValType>(column->size())); | 65 | 5.78k | } |
_ZNK5doris20Crc32HashPartitionerINS_24SpillPartitionChannelIdsEE8_do_hashERKNS_3COWINS_7IColumnEE13immutable_ptrIS4_EEPji Line | Count | Source | 61 | 8 | HashValType* __restrict result, int idx) const { | 62 | 8 | column->update_crcs_with_value( | 63 | 8 | result, _partition_expr_ctxs[idx]->root()->data_type()->get_primitive_type(), | 64 | 8 | cast_set<HashValType>(column->size())); | 65 | 8 | } |
_ZNK5doris20Crc32HashPartitionerINS_26SpillRePartitionChannelIdsEE8_do_hashERKNS_3COWINS_7IColumnEE13immutable_ptrIS4_EEPji Line | Count | Source | 61 | 8 | HashValType* __restrict result, int idx) const { | 62 | 8 | column->update_crcs_with_value( | 63 | 8 | result, _partition_expr_ctxs[idx]->root()->data_type()->get_primitive_type(), | 64 | 8 | cast_set<HashValType>(column->size())); | 65 | 8 | } |
Unexecuted instantiation: _ZNK5doris20Crc32HashPartitionerINS_15ShiftChannelIdsEE8_do_hashERKNS_3COWINS_7IColumnEE13immutable_ptrIS4_EEPji |
66 | | |
67 | | template <typename ChannelIds> |
68 | | Status Crc32HashPartitioner<ChannelIds>::clone(RuntimeState* state, |
69 | 4.83k | std::unique_ptr<PartitionerBase>& partitioner) { |
70 | 4.83k | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(_partition_count); |
71 | 4.83k | partitioner.reset(new_partitioner); |
72 | 4.83k | return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); |
73 | 4.83k | } _ZN5doris20Crc32HashPartitionerINS_17ShuffleChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS_15PartitionerBaseESt14default_deleteIS6_EE Line | Count | Source | 69 | 4.82k | std::unique_ptr<PartitionerBase>& partitioner) { | 70 | 4.82k | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(_partition_count); | 71 | 4.82k | partitioner.reset(new_partitioner); | 72 | 4.82k | return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); | 73 | 4.82k | } |
_ZN5doris20Crc32HashPartitionerINS_24SpillPartitionChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS_15PartitionerBaseESt14default_deleteIS6_EE Line | Count | Source | 69 | 5 | std::unique_ptr<PartitionerBase>& partitioner) { | 70 | 5 | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(_partition_count); | 71 | 5 | partitioner.reset(new_partitioner); | 72 | 5 | return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); | 73 | 5 | } |
_ZN5doris20Crc32HashPartitionerINS_26SpillRePartitionChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS_15PartitionerBaseESt14default_deleteIS6_EE Line | Count | Source | 69 | 4 | std::unique_ptr<PartitionerBase>& partitioner) { | 70 | 4 | auto* new_partitioner = new Crc32HashPartitioner<ChannelIds>(_partition_count); | 71 | 4 | partitioner.reset(new_partitioner); | 72 | 4 | return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); | 73 | 4 | } |
Unexecuted instantiation: _ZN5doris20Crc32HashPartitionerINS_15ShiftChannelIdsEE5cloneEPNS_12RuntimeStateERSt10unique_ptrINS_15PartitionerBaseESt14default_deleteIS6_EE |
74 | | |
75 | | void Crc32CHashPartitioner::_do_hash(const ColumnPtr& column, HashValType* __restrict result, |
76 | 68.6k | int idx) const { |
77 | 68.6k | column->update_crc32c_batch(result, nullptr); |
78 | 68.6k | } |
79 | | |
80 | | Status Crc32CHashPartitioner::clone(RuntimeState* state, |
81 | 76.5k | std::unique_ptr<PartitionerBase>& partitioner) { |
82 | 76.5k | auto* new_partitioner = new Crc32CHashPartitioner(_partition_count); |
83 | 76.5k | partitioner.reset(new_partitioner); |
84 | 76.5k | return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); |
85 | 76.5k | } |
86 | | |
87 | | template class Crc32HashPartitioner<ShuffleChannelIds>; |
88 | | template class Crc32HashPartitioner<SpillPartitionChannelIds>; |
89 | | template class Crc32HashPartitioner<SpillRePartitionChannelIds>; |
90 | | |
91 | | } // namespace doris |