be/src/cloud/delete_bitmap_file_writer.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "cloud/delete_bitmap_file_writer.h" |
19 | | |
20 | | #include <crc32c/crc32c.h> |
21 | | |
22 | | #include "cloud/config.h" |
23 | | #include "io/fs/file_writer.h" |
24 | | #include "io/fs/packed_file_writer.h" |
25 | | |
26 | | namespace doris { |
27 | | #include "common/compile_check_begin.h" |
28 | | |
29 | | DeleteBitmapFileWriter::DeleteBitmapFileWriter(int64_t tablet_id, const std::string& rowset_id, |
30 | | std::optional<StorageResource>& storage_resource) |
31 | 1 | : _tablet_id(tablet_id), _rowset_id(rowset_id), _storage_resource(storage_resource) {} |
32 | | |
33 | | DeleteBitmapFileWriter::DeleteBitmapFileWriter(int64_t tablet_id, const std::string& rowset_id, |
34 | | std::optional<StorageResource>& storage_resource, |
35 | | bool enable_packed_file, int64_t txn_id) |
36 | 0 | : _tablet_id(tablet_id), |
37 | 0 | _rowset_id(rowset_id), |
38 | 0 | _storage_resource(storage_resource), |
39 | 0 | _enable_packed_file(enable_packed_file), |
40 | 0 | _txn_id(txn_id) {} |
41 | | |
42 | 1 | DeleteBitmapFileWriter::~DeleteBitmapFileWriter() {} |
43 | | |
44 | 1 | Status DeleteBitmapFileWriter::init() { |
45 | 1 | #ifdef BE_TEST |
46 | 1 | _path = "./log/" + _rowset_id + "_delete_bitmap.db"; |
47 | 1 | io::Path path = _path; |
48 | 1 | auto parent_path = path.parent_path(); |
49 | 1 | bool exists = false; |
50 | 1 | RETURN_IF_ERROR(io::global_local_filesystem()->exists(parent_path, &exists)); |
51 | 1 | if (!exists) { |
52 | 0 | RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(parent_path)); |
53 | 0 | } |
54 | 1 | RETURN_IF_ERROR(io::global_local_filesystem()->create_file(_path, &_file_writer)); |
55 | 1 | return Status::OK(); |
56 | 0 | #endif |
57 | 0 | if (!_storage_resource) { |
58 | 0 | return Status::InternalError("invalid storage resource for tablet_id={}", _tablet_id); |
59 | 0 | } |
60 | 0 | _path = _storage_resource->remote_delete_bitmap_path(_tablet_id, _rowset_id); |
61 | 0 | io::FileWriterOptions opts; |
62 | |
|
63 | 0 | if (_enable_packed_file) { |
64 | | // Create underlying file writer |
65 | 0 | io::FileWriterPtr inner_writer; |
66 | | // Disable write_file_cache for inner writer when using PackedFileWriter. |
67 | | // Small files will be cached separately by PackedFileManager using the |
68 | | // small file path as cache key. |
69 | 0 | opts.write_file_cache = false; |
70 | 0 | RETURN_IF_ERROR(_storage_resource->fs->create_file(_path, &inner_writer, &opts)); |
71 | | |
72 | | // Wrap with PackedFileWriter |
73 | 0 | io::PackedAppendContext append_info; |
74 | 0 | append_info.resource_id = _storage_resource->fs->id(); |
75 | 0 | append_info.tablet_id = _tablet_id; |
76 | 0 | append_info.rowset_id = _rowset_id; |
77 | 0 | append_info.txn_id = _txn_id; |
78 | 0 | append_info.write_file_cache = false; |
79 | |
|
80 | 0 | _file_writer = std::make_unique<io::PackedFileWriter>(std::move(inner_writer), |
81 | 0 | io::Path(_path), append_info); |
82 | 0 | } else { |
83 | 0 | RETURN_IF_ERROR(_storage_resource->fs->create_file(_path, &_file_writer, &opts)); |
84 | 0 | } |
85 | 0 | return Status::OK(); |
86 | 0 | } |
87 | | |
88 | 1 | Status DeleteBitmapFileWriter::close() { |
89 | 1 | if (!_file_writer) { |
90 | 0 | return Status::InternalError("fail to close delete bitmap file={} because writer is null", |
91 | 0 | _path); |
92 | 0 | } |
93 | 1 | auto st = _file_writer->close(); |
94 | 1 | if (!st.ok()) { |
95 | 0 | LOG(WARNING) << "failed to close delete bitmap file=" << _path << ", st=" << st.to_string(); |
96 | 0 | return st; |
97 | 0 | } |
98 | | |
99 | | // Check if file was written to packed file |
100 | 1 | if (_enable_packed_file) { |
101 | 0 | auto* packed_writer = static_cast<io::PackedFileWriter*>(_file_writer.get()); |
102 | 0 | io::PackedSliceLocation loc; |
103 | 0 | st = packed_writer->get_packed_slice_location(&loc); |
104 | 0 | if (!st.ok()) { |
105 | 0 | LOG(WARNING) << "failed to get packed slice location for delete bitmap file=" << _path |
106 | 0 | << ", st=" << st.to_string(); |
107 | 0 | return st; |
108 | 0 | } |
109 | 0 | if (!loc.packed_file_path.empty()) { |
110 | 0 | _is_packed = true; |
111 | 0 | _packed_location = loc; |
112 | 0 | } |
113 | 0 | } |
114 | 1 | return Status::OK(); |
115 | 1 | } |
116 | | |
117 | 0 | Status DeleteBitmapFileWriter::get_packed_slice_location(io::PackedSliceLocation* location) const { |
118 | 0 | if (!_is_packed) { |
119 | 0 | return Status::InternalError("delete bitmap file is not packed"); |
120 | 0 | } |
121 | 0 | *location = _packed_location; |
122 | 0 | return Status::OK(); |
123 | 0 | } |
124 | | |
125 | 2 | Status DeleteBitmapFileWriter::write(const DeleteBitmapPB& delete_bitmap) { |
126 | 2 | if (delete_bitmap.rowset_ids_size() == 0) { |
127 | 1 | return Status::InternalError("empty delete bitmap for file={}", _path); |
128 | 1 | } |
129 | 1 | if (!_file_writer) { |
130 | 0 | return Status::InternalError("fail to write delete bitmap file={} because writer is null", |
131 | 0 | _path); |
132 | 0 | } |
133 | | // 0. write magic |
134 | 1 | RETURN_IF_ERROR(_file_writer->append({DELETE_BITMAP_MAGIC, MAGIC_SIZE})); |
135 | | |
136 | | // 1. write delete bitmap length |
137 | 1 | uint64_t delete_bitmap_len = delete_bitmap.ByteSizeLong(); |
138 | 1 | uint8_t len_buf[LENGTH_SIZE]; |
139 | 1 | encode_fixed64_le(len_buf, delete_bitmap_len); |
140 | 1 | RETURN_IF_ERROR(_file_writer->append({len_buf, LENGTH_SIZE})); |
141 | | |
142 | | // 2. write delete bitmap |
143 | 1 | std::string content = delete_bitmap.SerializeAsString(); |
144 | 1 | RETURN_IF_ERROR(_file_writer->append(content)); |
145 | | |
146 | | // 3. write checksum |
147 | 1 | uint8_t checksum_buf[CHECKSUM_SIZE]; |
148 | 1 | uint32_t checksum = crc32c::Crc32c(content.data(), delete_bitmap_len); |
149 | 1 | encode_fixed32_le(checksum_buf, checksum); |
150 | 1 | RETURN_IF_ERROR(_file_writer->append({checksum_buf, CHECKSUM_SIZE})); |
151 | 1 | return Status::OK(); |
152 | 1 | } |
153 | | |
154 | | #include "common/compile_check_end.h" |
155 | | } // namespace doris |