be/src/service/http/action/check_encryption_action.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "service/http/action/check_encryption_action.h" |
19 | | |
20 | | #include <gen_cpp/olap_file.pb.h> |
21 | | #include <glog/logging.h> |
22 | | #include <google/protobuf/util/json_util.h> |
23 | | #include <json2pb/pb_to_json.h> |
24 | | |
25 | | #include <cstdint> |
26 | | #include <exception> |
27 | | #include <memory> |
28 | | #include <shared_mutex> |
29 | | #include <string> |
30 | | #include <string_view> |
31 | | |
32 | | #include "cloud/cloud_tablet.h" |
33 | | #include "cloud/config.h" |
34 | | #include "common/status.h" |
35 | | #include "io/fs/file_reader.h" |
36 | | #include "io/fs/file_system.h" |
37 | | #include "io/fs/path.h" |
38 | | #include "runtime/exec_env.h" |
39 | | #include "service/http/http_channel.h" |
40 | | #include "service/http/http_headers.h" |
41 | | #include "service/http/http_status.h" |
42 | | #include "storage/rowset/rowset_fwd.h" |
43 | | #include "storage/tablet/tablet_fwd.h" |
44 | | |
45 | | namespace doris { |
46 | | |
47 | | const std::string TABLET_ID = "tablet_id"; |
48 | | const std::string GET_FOOTER = "get_footer"; |
49 | | |
50 | | CheckEncryptionAction::CheckEncryptionAction(ExecEnv* exec_env, TPrivilegeHier::type hier, |
51 | | TPrivilegeType::type type) |
52 | 0 | : HttpHandlerWithAuth(exec_env, hier, type) {} |
53 | | |
54 | 0 | Result<bool> is_tablet_encrypted(const BaseTabletSPtr& tablet) { |
55 | 0 | auto tablet_meta = tablet->tablet_meta(); |
56 | 0 | if (tablet_meta->encryption_algorithm() == EncryptionAlgorithmPB::PLAINTEXT) { |
57 | 0 | return false; |
58 | 0 | } |
59 | 0 | Status st; |
60 | 0 | bool is_encrypted = true; |
61 | 0 | tablet->traverse_rowsets([&st, &tablet, &is_encrypted](const RowsetSharedPtr& rs) { |
62 | 0 | if (!st) { |
63 | 0 | return; |
64 | 0 | } |
65 | | |
66 | 0 | auto rs_meta = rs->rowset_meta(); |
67 | 0 | if (config::is_cloud_mode() && rs_meta->start_version() == 0 && |
68 | 0 | rs_meta->end_version() == 1) { |
69 | 0 | return; |
70 | 0 | } |
71 | 0 | auto fs = rs_meta->physical_fs(); |
72 | 0 | if (fs == nullptr) { |
73 | 0 | st = Status::InternalError("failed to get fs for rowset: tablet={}, rs={}", |
74 | 0 | tablet->tablet_id(), rs->rowset_id().to_string()); |
75 | 0 | return; |
76 | 0 | } |
77 | | |
78 | 0 | if (rs->num_segments() == 0) { |
79 | 0 | return; |
80 | 0 | } |
81 | 0 | auto maybe_seg_path = rs->segment_path(0); |
82 | 0 | if (!maybe_seg_path) { |
83 | 0 | st = std::move(maybe_seg_path.error()); |
84 | 0 | return; |
85 | 0 | } |
86 | | |
87 | 0 | std::vector<std::string_view> file_paths; |
88 | 0 | const auto& first_seg_path = maybe_seg_path.value(); |
89 | 0 | file_paths.emplace_back(first_seg_path); |
90 | 0 | if (tablet->tablet_schema()->has_inverted_index() && |
91 | 0 | tablet->tablet_schema()->get_inverted_index_storage_format() == V2) { |
92 | 0 | std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_path_v2( |
93 | 0 | InvertedIndexDescriptor::get_index_file_path_prefix(first_seg_path)); |
94 | 0 | file_paths.emplace_back(inverted_index_file_path); |
95 | 0 | } |
96 | |
|
97 | 0 | for (const auto path : file_paths) { |
98 | 0 | io::FileReaderSPtr reader; |
99 | 0 | st = fs->open_file(path, &reader); |
100 | 0 | if (!st) { |
101 | 0 | return; |
102 | 0 | } |
103 | 0 | std::vector<uint8_t> magic_code_buf; |
104 | 0 | magic_code_buf.resize(sizeof(uint64_t)); |
105 | 0 | Slice magic_code(magic_code_buf.data(), sizeof(uint64_t)); |
106 | 0 | size_t bytes_read; |
107 | 0 | st = reader->read_at(reader->size() - sizeof(uint64_t), magic_code, &bytes_read); |
108 | 0 | if (!st) { |
109 | 0 | return; |
110 | 0 | } |
111 | | |
112 | 0 | std::vector<uint8_t> answer = {'A', 'B', 'C', 'D', 'E', 'A', 'B', 'C'}; |
113 | 0 | is_encrypted &= Slice::mem_equal(answer.data(), magic_code.data, magic_code.size); |
114 | 0 | if (!is_encrypted) { |
115 | 0 | LOG(INFO) << "found not encrypted segment, path=" << first_seg_path; |
116 | 0 | } |
117 | 0 | } |
118 | 0 | }); |
119 | |
|
120 | 0 | if (st) { |
121 | 0 | return is_encrypted; |
122 | 0 | } |
123 | 0 | return st; |
124 | 0 | } |
125 | | |
126 | 0 | Result<std::string> get_last_encrypt_footer(const BaseTabletSPtr& tablet) { |
127 | 0 | std::shared_lock l(tablet->get_header_lock()); |
128 | 0 | auto rs = tablet->get_rowset_with_max_version(); |
129 | 0 | if (rs->num_segments() == 0) { |
130 | 0 | return "{}"; |
131 | 0 | } |
132 | 0 | auto maybe_seg_path = rs->segment_path(0); |
133 | 0 | if (!maybe_seg_path) { |
134 | 0 | return ResultError(maybe_seg_path.error()); |
135 | 0 | } |
136 | 0 | auto rs_meta = rs->rowset_meta(); |
137 | 0 | if (config::is_cloud_mode() && rs_meta->start_version() == 0 && rs_meta->end_version() == 1) { |
138 | 0 | return "{}"; |
139 | 0 | } |
140 | 0 | auto fs = rs_meta->physical_fs(); |
141 | 0 | io::FileReaderSPtr reader; |
142 | 0 | RETURN_IF_ERROR_RESULT(fs->open_file(maybe_seg_path.value(), &reader)); |
143 | | |
144 | 0 | std::vector<uint8_t> pb_len_buf; |
145 | 0 | pb_len_buf.reserve(sizeof(uint64_t)); |
146 | 0 | Slice pb_len_slice(pb_len_buf.data(), sizeof(uint64_t)); |
147 | 0 | size_t bytes_read; |
148 | 0 | RETURN_IF_ERROR_RESULT( |
149 | 0 | reader->read_at(reader->size() - 256 + sizeof(uint8_t), pb_len_slice, &bytes_read)); |
150 | 0 | auto info_pb_size = decode_fixed64_le(pb_len_buf.data()); |
151 | |
|
152 | 0 | std::vector<uint8_t> info_pb_buf; |
153 | 0 | info_pb_buf.resize(info_pb_size); |
154 | 0 | Slice pb_slice(info_pb_buf.data(), info_pb_size); |
155 | 0 | RETURN_IF_ERROR_RESULT(reader->read_at( |
156 | 0 | reader->size() - 256 + sizeof(uint8_t) + sizeof(uint64_t), pb_slice, &bytes_read)); |
157 | | |
158 | 0 | FileEncryptionInfoPB info_pb; |
159 | 0 | if (!info_pb.ParseFromArray(info_pb_buf.data(), static_cast<int>(info_pb_buf.size()))) { |
160 | 0 | return ResultError(Status::Corruption("parse encryption info failed")); |
161 | 0 | } |
162 | 0 | std::string json; |
163 | 0 | google::protobuf::util::JsonPrintOptions opts; |
164 | 0 | opts.add_whitespace = false; |
165 | 0 | opts.preserve_proto_field_names = true; |
166 | 0 | auto st = google::protobuf::util::MessageToJsonString(info_pb, &json, opts); |
167 | 0 | return json; |
168 | 0 | } |
169 | | |
170 | 0 | Status sync_meta(const CloudTabletSPtr& tablet) { |
171 | 0 | RETURN_IF_ERROR(tablet->sync_meta()); |
172 | 0 | RETURN_IF_ERROR(tablet->sync_rowsets()); |
173 | 0 | return Status::OK(); |
174 | 0 | } |
175 | | |
176 | 0 | void CheckEncryptionAction::handle(HttpRequest* req) { |
177 | 0 | req->add_output_header(HttpHeaders::CONTENT_TYPE, HttpHeaders::JSON_TYPE.data()); |
178 | 0 | auto tablet_id_str = req->param(TABLET_ID); |
179 | |
|
180 | 0 | if (tablet_id_str.empty()) { |
181 | 0 | HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, |
182 | 0 | "tablet id should be set in request params"); |
183 | 0 | return; |
184 | 0 | } |
185 | 0 | int64_t tablet_id = -1; |
186 | 0 | try { |
187 | 0 | tablet_id = std::stoll(tablet_id_str); |
188 | 0 | } catch (const std::exception& e) { |
189 | 0 | LOG(WARNING) << "convert tablet id to i64 failed:" << e.what(); |
190 | 0 | auto msg = fmt::format("invalid argument: tablet_id={}", tablet_id_str); |
191 | |
|
192 | 0 | HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, msg); |
193 | 0 | return; |
194 | 0 | } |
195 | | |
196 | 0 | bool is_get_footer = false; |
197 | 0 | if (auto get_footer_flag = req->param(GET_FOOTER); get_footer_flag == "true") { |
198 | 0 | is_get_footer = true; |
199 | 0 | } else if (get_footer_flag != "false") { |
200 | 0 | HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, |
201 | 0 | "param `get_footer` must be a boolean type"); |
202 | 0 | return; |
203 | 0 | } |
204 | | |
205 | 0 | auto maybe_tablet = ExecEnv::get_tablet(tablet_id); |
206 | 0 | if (!maybe_tablet) { |
207 | 0 | HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, maybe_tablet.error().to_string()); |
208 | 0 | return; |
209 | 0 | } |
210 | 0 | auto tablet = maybe_tablet.value(); |
211 | |
|
212 | 0 | if (config::is_cloud_mode()) { |
213 | 0 | auto cloud_tablet = std::dynamic_pointer_cast<CloudTablet>(tablet); |
214 | 0 | DCHECK_NE(cloud_tablet, nullptr); |
215 | 0 | auto st = sync_meta(cloud_tablet); |
216 | 0 | if (!st) { |
217 | 0 | HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, st.to_json()); |
218 | 0 | return; |
219 | 0 | } |
220 | 0 | } |
221 | | |
222 | 0 | auto maybe_is_encrypted = is_tablet_encrypted(tablet); |
223 | 0 | if (maybe_is_encrypted.has_value()) { |
224 | 0 | req->add_output_header(HttpHeaders::CONTENT_TYPE, HttpHeaders::JSON_TYPE.data()); |
225 | 0 | std::string result = R"({"status":)"; |
226 | 0 | result += maybe_is_encrypted.value() ? R"("all encrypted")" : R"("some are not encrypted")"; |
227 | 0 | if (is_get_footer) { |
228 | 0 | auto maybe_footer = get_last_encrypt_footer(tablet); |
229 | 0 | if (!maybe_footer) { |
230 | 0 | HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, |
231 | 0 | maybe_footer.error().to_json()); |
232 | 0 | return; |
233 | 0 | } |
234 | 0 | result += R"(,"footer":)"; |
235 | 0 | result += maybe_footer.value(); |
236 | 0 | } |
237 | 0 | result += "}"; |
238 | |
|
239 | 0 | HttpChannel::send_reply(req, HttpStatus::OK, result); |
240 | 0 | return; |
241 | 0 | } |
242 | 0 | HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, |
243 | 0 | maybe_is_encrypted.error().to_json()); |
244 | 0 | } |
245 | | |
246 | | } // namespace doris |