be/src/cloud/delete_bitmap_file_writer.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "cloud/delete_bitmap_file_writer.h" |
19 | | |
20 | | #include <crc32c/crc32c.h> |
21 | | |
22 | | #include "cloud/config.h" |
23 | | #include "io/fs/file_writer.h" |
24 | | #include "io/fs/packed_file_writer.h" |
25 | | |
26 | | namespace doris { |
27 | | |
28 | | DeleteBitmapFileWriter::DeleteBitmapFileWriter(int64_t tablet_id, const std::string& rowset_id, |
29 | | std::optional<StorageResource>& storage_resource) |
30 | 1 | : _tablet_id(tablet_id), _rowset_id(rowset_id), _storage_resource(storage_resource) {} |
31 | | |
32 | | DeleteBitmapFileWriter::DeleteBitmapFileWriter(int64_t tablet_id, const std::string& rowset_id, |
33 | | std::optional<StorageResource>& storage_resource, |
34 | | bool enable_packed_file, int64_t txn_id) |
35 | 0 | : _tablet_id(tablet_id), |
36 | 0 | _rowset_id(rowset_id), |
37 | 0 | _storage_resource(storage_resource), |
38 | 0 | _enable_packed_file(enable_packed_file), |
39 | 0 | _txn_id(txn_id) {} |
40 | | |
41 | 1 | DeleteBitmapFileWriter::~DeleteBitmapFileWriter() {} |
42 | | |
43 | 0 | Status DeleteBitmapFileWriter::init() { |
44 | | #ifdef BE_TEST |
45 | | _path = "./log/" + _rowset_id + "_delete_bitmap.db"; |
46 | | io::Path path = _path; |
47 | | auto parent_path = path.parent_path(); |
48 | | bool exists = false; |
49 | | RETURN_IF_ERROR(io::global_local_filesystem()->exists(parent_path, &exists)); |
50 | | if (!exists) { |
51 | | RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(parent_path)); |
52 | | } |
53 | | RETURN_IF_ERROR(io::global_local_filesystem()->create_file(_path, &_file_writer)); |
54 | | return Status::OK(); |
55 | | #endif |
56 | 0 | if (!_storage_resource) { |
57 | 0 | return Status::InternalError("invalid storage resource for tablet_id={}", _tablet_id); |
58 | 0 | } |
59 | 0 | _path = _storage_resource->remote_delete_bitmap_path(_tablet_id, _rowset_id); |
60 | 0 | io::FileWriterOptions opts; |
61 | |
|
62 | 0 | if (_enable_packed_file) { |
63 | | // Create underlying file writer |
64 | 0 | io::FileWriterPtr inner_writer; |
65 | | // Disable write_file_cache for inner writer when using PackedFileWriter. |
66 | | // Small files will be cached separately by PackedFileManager using the |
67 | | // small file path as cache key. |
68 | 0 | opts.write_file_cache = false; |
69 | 0 | RETURN_IF_ERROR(_storage_resource->fs->create_file(_path, &inner_writer, &opts)); |
70 | | |
71 | | // Wrap with PackedFileWriter |
72 | 0 | io::PackedAppendContext append_info; |
73 | 0 | append_info.resource_id = _storage_resource->fs->id(); |
74 | 0 | append_info.tablet_id = _tablet_id; |
75 | 0 | append_info.rowset_id = _rowset_id; |
76 | 0 | append_info.txn_id = _txn_id; |
77 | 0 | append_info.write_file_cache = false; |
78 | |
|
79 | 0 | _file_writer = std::make_unique<io::PackedFileWriter>(std::move(inner_writer), |
80 | 0 | io::Path(_path), append_info); |
81 | 0 | } else { |
82 | 0 | RETURN_IF_ERROR(_storage_resource->fs->create_file(_path, &_file_writer, &opts)); |
83 | 0 | } |
84 | 0 | return Status::OK(); |
85 | 0 | } |
86 | | |
87 | 1 | Status DeleteBitmapFileWriter::close() { |
88 | 1 | if (!_file_writer) { |
89 | 0 | return Status::InternalError("fail to close delete bitmap file={} because writer is null", |
90 | 0 | _path); |
91 | 0 | } |
92 | 1 | auto st = _file_writer->close(); |
93 | 1 | if (!st.ok()) { |
94 | 0 | LOG(WARNING) << "failed to close delete bitmap file=" << _path << ", st=" << st.to_string(); |
95 | 0 | return st; |
96 | 0 | } |
97 | | |
98 | | // Check if file was written to packed file |
99 | 1 | if (_enable_packed_file) { |
100 | 0 | auto* packed_writer = static_cast<io::PackedFileWriter*>(_file_writer.get()); |
101 | 0 | io::PackedSliceLocation loc; |
102 | 0 | st = packed_writer->get_packed_slice_location(&loc); |
103 | 0 | if (!st.ok()) { |
104 | 0 | LOG(WARNING) << "failed to get packed slice location for delete bitmap file=" << _path |
105 | 0 | << ", st=" << st.to_string(); |
106 | 0 | return st; |
107 | 0 | } |
108 | 0 | if (!loc.packed_file_path.empty()) { |
109 | 0 | _is_packed = true; |
110 | 0 | _packed_location = loc; |
111 | 0 | } |
112 | 0 | } |
113 | 1 | return Status::OK(); |
114 | 1 | } |
115 | | |
116 | 0 | Status DeleteBitmapFileWriter::get_packed_slice_location(io::PackedSliceLocation* location) const { |
117 | 0 | if (!_is_packed) { |
118 | 0 | return Status::InternalError("delete bitmap file is not packed"); |
119 | 0 | } |
120 | 0 | *location = _packed_location; |
121 | 0 | return Status::OK(); |
122 | 0 | } |
123 | | |
124 | 2 | Status DeleteBitmapFileWriter::write(const DeleteBitmapPB& delete_bitmap) { |
125 | 2 | if (delete_bitmap.rowset_ids_size() == 0) { |
126 | 1 | return Status::InternalError("empty delete bitmap for file={}", _path); |
127 | 1 | } |
128 | 1 | if (!_file_writer) { |
129 | 0 | return Status::InternalError("fail to write delete bitmap file={} because writer is null", |
130 | 0 | _path); |
131 | 0 | } |
132 | | // 0. write magic |
133 | 1 | RETURN_IF_ERROR(_file_writer->append({DELETE_BITMAP_MAGIC, MAGIC_SIZE})); |
134 | | |
135 | | // 1. write delete bitmap length |
136 | 1 | uint64_t delete_bitmap_len = delete_bitmap.ByteSizeLong(); |
137 | 1 | uint8_t len_buf[LENGTH_SIZE]; |
138 | 1 | encode_fixed64_le(len_buf, delete_bitmap_len); |
139 | 1 | RETURN_IF_ERROR(_file_writer->append({len_buf, LENGTH_SIZE})); |
140 | | |
141 | | // 2. write delete bitmap |
142 | 1 | std::string content = delete_bitmap.SerializeAsString(); |
143 | 1 | RETURN_IF_ERROR(_file_writer->append(content)); |
144 | | |
145 | | // 3. write checksum |
146 | 1 | uint8_t checksum_buf[CHECKSUM_SIZE]; |
147 | 1 | uint32_t checksum = crc32c::Crc32c(content.data(), delete_bitmap_len); |
148 | 1 | encode_fixed32_le(checksum_buf, checksum); |
149 | 1 | RETURN_IF_ERROR(_file_writer->append({checksum_buf, CHECKSUM_SIZE})); |
150 | 1 | return Status::OK(); |
151 | 1 | } |
152 | | |
153 | | } // namespace doris |