Coverage Report

Created: 2024-11-21 14:00

/root/doris/be/src/olap/rowset/segcompaction.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <memory>
21
#include <vector>
22
23
#include "common/status.h"
24
#include "io/fs/file_reader_writer_fwd.h"
25
#include "olap/merger.h"
26
#include "olap/simple_rowid_conversion.h"
27
#include "olap/tablet.h"
28
#include "segment_v2/inverted_index_file_writer.h"
29
#include "segment_v2/segment.h"
30
31
namespace doris {
32
class Schema;
33
34
namespace segment_v2 {
35
class SegmentWriter;
36
} // namespace segment_v2
37
namespace vectorized {
38
class RowSourcesBuffer;
39
class VerticalBlockReader;
40
} // namespace vectorized
41
struct OlapReaderStatistics;
42
43
using SegCompactionCandidates = std::vector<segment_v2::SegmentSharedPtr>;
44
using SegCompactionCandidatesSharedPtr = std::shared_ptr<SegCompactionCandidates>;
45
46
class BetaRowsetWriter;
47
48
class SegcompactionWorker {
49
    friend class BetaRowsetWriter;
50
51
public:
52
    explicit SegcompactionWorker(BetaRowsetWriter* writer);
53
54
472
    ~SegcompactionWorker() {
55
472
        DCHECK(_seg_compact_mem_tracker != nullptr);
56
472
        SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_seg_compact_mem_tracker);
57
472
        if (_rowid_conversion) {
58
3
            _rowid_conversion.reset();
59
3
        }
60
472
    }
61
62
    void compact_segments(SegCompactionCandidatesSharedPtr segments);
63
64
    bool need_convert_delete_bitmap();
65
66
    void convert_segment_delete_bitmap(DeleteBitmapPtr src_delete_bitmap, uint32_t src_seg_id,
67
                                       uint32_t dest_seg_id);
68
    void convert_segment_delete_bitmap(DeleteBitmapPtr src_delete_bitmap, uint32_t src_begin,
69
                                       uint32_t src_end, uint32_t dest_seg_id);
70
4
    DeleteBitmapPtr get_converted_delete_bitmap() { return _converted_delete_bitmap; }
71
72
203
    io::FileWriterPtr& get_file_writer() { return _file_writer; }
73
24
    InvertedIndexFileWriterPtr& get_inverted_index_file_writer() {
74
24
        return _inverted_index_file_writer;
75
24
    }
76
77
    // set the cancel flag, tasks already started will not be cancelled.
78
    bool cancel();
79
80
    void init_mem_tracker(const RowsetWriterContext& rowset_writer_context);
81
82
private:
83
    Status _create_segment_writer_for_segcompaction(
84
            std::unique_ptr<segment_v2::SegmentWriter>* writer, uint32_t begin, uint32_t end);
85
    Status _get_segcompaction_reader(SegCompactionCandidatesSharedPtr segments,
86
                                     TabletSharedPtr tablet, std::shared_ptr<Schema> schema,
87
                                     OlapReaderStatistics* stat,
88
                                     vectorized::RowSourcesBuffer& row_sources_buf, bool is_key,
89
                                     std::vector<uint32_t>& return_columns,
90
                                     std::unique_ptr<vectorized::VerticalBlockReader>* reader);
91
    std::unique_ptr<segment_v2::SegmentWriter> _create_segcompaction_writer(uint32_t begin,
92
                                                                            uint32_t end);
93
    Status _delete_original_segments(uint32_t begin, uint32_t end);
94
    Status _check_correctness(OlapReaderStatistics& reader_stat, Merger::Statistics& merger_stat,
95
                              uint32_t begin, uint32_t end);
96
    Status _do_compact_segments(SegCompactionCandidatesSharedPtr segments);
97
98
private:
99
    //TODO(zhengyu): current impl depends heavily on the access to feilds of BetaRowsetWriter
100
    // Currently cloud storage engine doesn't need segcompaction
101
    BetaRowsetWriter* _writer = nullptr;
102
    io::FileWriterPtr _file_writer;
103
    InvertedIndexFileWriterPtr _inverted_index_file_writer = nullptr;
104
105
    // for unique key mow table
106
    std::unique_ptr<SimpleRowIdConversion> _rowid_conversion = nullptr;
107
    DeleteBitmapPtr _converted_delete_bitmap;
108
    std::shared_ptr<MemTrackerLimiter> _seg_compact_mem_tracker = nullptr;
109
110
    // the state is not mutable when 1)actual compaction operation started or 2) cancelled
111
    std::atomic<bool> _is_compacting_state_mutable = true;
112
};
113
} // namespace doris