/root/doris/be/src/olap/tablet.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <butil/macros.h> |
21 | | #include <glog/logging.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <cstddef> |
25 | | #include <cstdint> |
26 | | #include <functional> |
27 | | #include <memory> |
28 | | #include <mutex> |
29 | | #include <ostream> |
30 | | #include <set> |
31 | | #include <shared_mutex> |
32 | | #include <string> |
33 | | #include <string_view> |
34 | | #include <utility> |
35 | | #include <vector> |
36 | | |
37 | | #include "common/config.h" |
38 | | #include "common/status.h" |
39 | | #include "olap/base_tablet.h" |
40 | | #include "olap/binlog_config.h" |
41 | | #include "olap/data_dir.h" |
42 | | #include "olap/olap_common.h" |
43 | | #include "olap/partial_update_info.h" |
44 | | #include "olap/rowset/rowset.h" |
45 | | #include "olap/rowset/rowset_meta.h" |
46 | | #include "olap/rowset/rowset_reader.h" |
47 | | #include "olap/rowset/segment_v2/segment.h" |
48 | | #include "olap/version_graph.h" |
49 | | #include "segment_loader.h" |
50 | | #include "util/metrics.h" |
51 | | #include "util/once.h" |
52 | | #include "util/slice.h" |
53 | | |
54 | | namespace bvar { |
55 | | template <typename T> |
56 | | class Adder; |
57 | | } |
58 | | |
59 | | namespace doris { |
60 | | #include "common/compile_check_begin.h" |
61 | | |
62 | | class Tablet; |
63 | | class CumulativeCompactionPolicy; |
64 | | class CompactionMixin; |
65 | | class SingleReplicaCompaction; |
66 | | class RowsetWriter; |
67 | | struct RowsetWriterContext; |
68 | | class TTabletInfo; |
69 | | class TabletMetaPB; |
70 | | class TupleDescriptor; |
71 | | class CalcDeleteBitmapToken; |
72 | | enum CompressKind : int; |
73 | | class RowsetBinlogMetasPB; |
74 | | |
75 | | namespace io { |
76 | | class RemoteFileSystem; |
77 | | } // namespace io |
78 | | namespace vectorized { |
79 | | class Block; |
80 | | } // namespace vectorized |
81 | | struct RowLocation; |
82 | | enum KeysType : int; |
83 | | enum SortType : int; |
84 | | |
85 | | enum TabletStorageType { STORAGE_TYPE_LOCAL, STORAGE_TYPE_REMOTE, STORAGE_TYPE_REMOTE_AND_LOCAL }; |
86 | | |
87 | | extern bvar::Adder<uint64_t> unused_remote_rowset_num; |
88 | | |
89 | | static inline constexpr auto TRACE_TABLET_LOCK_THRESHOLD = std::chrono::seconds(1); |
90 | | |
91 | | struct WriteCooldownMetaExecutors { |
92 | | WriteCooldownMetaExecutors(size_t executor_nums = 5); |
93 | | |
94 | | void stop(); |
95 | | |
96 | | void submit(TabletSharedPtr tablet); |
97 | 301k | size_t _get_executor_pos(int64_t tablet_id) const { |
98 | 301k | return std::hash<int64_t>()(tablet_id) % _executor_nums; |
99 | 301k | }; |
100 | | // Each executor is a mpsc to ensure uploads of the same tablet meta are not concurrent |
101 | | // FIXME(AlexYue): Use mpsc instead of `ThreadPool` with 1 thread |
102 | | // We use PriorityThreadPool since it would call status inside it's `shutdown` function. |
103 | | // Consider one situation where the StackTraceCache's singleton is detructed before |
104 | | // this WriteCooldownMetaExecutors's singleton, then invoking the status would also call |
105 | | // StackTraceCache which would then result in heap use after free like #23834 |
106 | | std::vector<std::unique_ptr<PriorityThreadPool>> _executors; |
107 | | std::unordered_set<int64_t> _pending_tablets; |
108 | | std::mutex _latch; |
109 | | size_t _executor_nums; |
110 | | }; |
111 | | |
112 | | class Tablet final : public BaseTablet { |
113 | | public: |
114 | | Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* data_dir, |
115 | | const std::string_view& cumulative_compaction_type = ""); |
116 | | |
117 | 37.3M | DataDir* data_dir() const { return _data_dir; } |
118 | 1.62M | int64_t replica_id() const { return _tablet_meta->replica_id(); } |
119 | | |
120 | 65.7k | std::string tablet_path() const override { return _tablet_path; } |
121 | | |
122 | | bool set_tablet_schema_into_rowset_meta(); |
123 | | Status init(); |
124 | | bool init_succeeded(); |
125 | | |
126 | | bool is_used(); |
127 | | |
128 | | void register_tablet_into_dir(); |
129 | | void deregister_tablet_from_dir(); |
130 | | |
131 | | void save_meta(); |
132 | | // Used in clone task, to update local meta when finishing a clone job |
133 | | Status revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add, |
134 | | const std::vector<RowsetSharedPtr>& to_delete, |
135 | | bool is_incremental_clone); |
136 | | |
137 | | int64_t cumulative_layer_point() const; |
138 | | void set_cumulative_layer_point(int64_t new_point); |
139 | | inline int64_t cumulative_promotion_size() const; |
140 | | inline void set_cumulative_promotion_size(int64_t new_size); |
141 | | |
142 | | // Disk space occupied by tablet, contain local and remote. |
143 | | size_t tablet_footprint() override; |
144 | | // Local disk space occupied by tablet. |
145 | | size_t tablet_local_size(); |
146 | | // Remote disk space occupied by tablet. |
147 | | size_t tablet_remote_size(); |
148 | | |
149 | | size_t num_rows(); |
150 | | size_t version_count() const; |
151 | | size_t stale_version_count() const; |
152 | | bool exceed_version_limit(int32_t limit) override; |
153 | | uint64_t segment_count() const; |
154 | | Version max_version() const; |
155 | | CumulativeCompactionPolicy* cumulative_compaction_policy(); |
156 | | |
157 | | // properties encapsulated in TabletSchema |
158 | | SortType sort_type() const; |
159 | | size_t sort_col_num() const; |
160 | | size_t num_columns() const; |
161 | | size_t num_null_columns() const; |
162 | | size_t num_short_key_columns() const; |
163 | | size_t num_rows_per_row_block() const; |
164 | | double bloom_filter_fpp() const; |
165 | | size_t next_unique_id() const; |
166 | | int64_t avg_rs_meta_serialize_size() const; |
167 | | |
168 | | // operation in rowsets |
169 | | Status add_rowset(RowsetSharedPtr rowset); |
170 | | Status create_initial_rowset(const int64_t version); |
171 | | |
172 | | // MUST hold EXCLUSIVE `_meta_lock`. |
173 | | Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add, |
174 | | std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false); |
175 | | bool rowset_exists_unlocked(const RowsetSharedPtr& rowset); |
176 | | |
177 | | Status add_inc_rowset(const RowsetSharedPtr& rowset); |
178 | | /// Delete stale rowset by timing. This delete policy uses now() minutes |
179 | | /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset |
180 | | /// to delete. When rowset is deleted, it will be added to StorageEngine unused map and record |
181 | | /// need to delete flag. |
182 | | void delete_expired_stale_rowset(); |
183 | | |
184 | | // Given spec_version, find a continuous version path and store it in version_path. |
185 | | // If quiet is true, then only "does this path exist" is returned. |
186 | | // If skip_missing_version is true, return ok even there are missing versions. |
187 | | Status capture_consistent_versions_unlocked(const Version& spec_version, Versions* version_path, |
188 | | bool skip_missing_version, bool quiet) const; |
189 | | |
190 | | // if quiet is true, no error log will be printed if there are missing versions |
191 | | Status check_version_integrity(const Version& version, bool quiet = false); |
192 | | bool check_version_exist(const Version& version) const; |
193 | | void acquire_version_and_rowsets( |
194 | | std::vector<std::pair<Version, RowsetSharedPtr>>* version_rowsets) const; |
195 | | |
196 | | Status capture_consistent_rowsets_unlocked( |
197 | | const Version& spec_version, std::vector<RowsetSharedPtr>* rowsets) const override; |
198 | | |
199 | | // If skip_missing_version is true, skip versions if they are missing. |
200 | | Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits, |
201 | | bool skip_missing_version) override; |
202 | | |
203 | | // Find the missed versions until the spec_version. |
204 | | // |
205 | | // for example: |
206 | | // [0-4][5-5][8-8][9-9][14-14] |
207 | | // if spec_version = 12, it will return [6, 6], [7, 7], [10, 10], [11, 11], [12, 12] |
208 | | Versions calc_missed_versions(int64_t spec_version, Versions existing_versions) const override; |
209 | | |
210 | | // meta lock |
211 | 370k | std::shared_mutex& get_header_lock() { return _meta_lock; } |
212 | 412 | std::mutex& get_rowset_update_lock() { return _rowset_update_lock; } |
213 | 414 | std::mutex& get_push_lock() { return _ingest_lock; } |
214 | 7.72M | std::mutex& get_base_compaction_lock() { return _base_compaction_lock; } |
215 | 10.1M | std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; } |
216 | 1 | std::shared_mutex& get_meta_store_lock() { return _meta_store_lock; } |
217 | | |
218 | 816 | std::shared_timed_mutex& get_migration_lock() { return _migration_lock; } |
219 | | |
220 | 19 | std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; } |
221 | | |
222 | | // operation for compaction |
223 | | bool can_do_compaction(size_t path_hash, CompactionType compaction_type); |
224 | | bool suitable_for_compaction( |
225 | | CompactionType compaction_type, |
226 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
227 | | |
228 | | uint32_t calc_compaction_score(); |
229 | | |
230 | | // This function to find max continuous version from the beginning. |
231 | | // For example: If there are 1, 2, 3, 5, 6, 7 versions belongs tablet, then 3 is target. |
232 | | // 3 will be saved in "version", and 7 will be saved in "max_version", if max_version != nullptr |
233 | | void max_continuous_version_from_beginning(Version* version, Version* max_version = nullptr); |
234 | | |
235 | 0 | void set_bad(bool is_bad) { _is_bad = is_bad; } |
236 | | |
237 | 18.1M | int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; } |
238 | 47.2k | void set_last_cumu_compaction_failure_time(int64_t millis) { |
239 | 47.2k | _last_cumu_compaction_failure_millis = millis; |
240 | 47.2k | } |
241 | | |
242 | 7.71M | int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; } |
243 | 19.3k | void set_last_base_compaction_failure_time(int64_t millis) { |
244 | 19.3k | _last_base_compaction_failure_millis = millis; |
245 | 19.3k | } |
246 | | |
247 | 0 | int64_t last_full_compaction_failure_time() { return _last_full_compaction_failure_millis; } |
248 | 0 | void set_last_full_compaction_failure_time(int64_t millis) { |
249 | 0 | _last_full_compaction_failure_millis = millis; |
250 | 0 | } |
251 | | |
252 | 41.7k | int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; } |
253 | 1.50k | void set_last_cumu_compaction_success_time(int64_t millis) { |
254 | 1.50k | _last_cumu_compaction_success_millis = millis; |
255 | 1.50k | } |
256 | | |
257 | 41.7k | int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; } |
258 | 1.46k | void set_last_base_compaction_success_time(int64_t millis) { |
259 | 1.46k | _last_base_compaction_success_millis = millis; |
260 | 1.46k | } |
261 | | |
262 | 0 | int64_t last_full_compaction_success_time() { return _last_full_compaction_success_millis; } |
263 | 0 | void set_last_full_compaction_success_time(int64_t millis) { |
264 | 0 | _last_full_compaction_success_millis = millis; |
265 | 0 | } |
266 | | |
267 | 0 | int64_t last_cumu_compaction_schedule_time() { return _last_cumu_compaction_schedule_millis; } |
268 | 47.2k | void set_last_cumu_compaction_schedule_time(int64_t millis) { |
269 | 47.2k | _last_cumu_compaction_schedule_millis = millis; |
270 | 47.2k | } |
271 | | |
272 | 0 | int64_t last_base_compaction_schedule_time() { return _last_base_compaction_schedule_millis; } |
273 | 9.67k | void set_last_base_compaction_schedule_time(int64_t millis) { |
274 | 9.67k | _last_base_compaction_schedule_millis = millis; |
275 | 9.67k | } |
276 | | |
277 | 0 | int64_t last_full_compaction_schedule_time() { return _last_full_compaction_schedule_millis; } |
278 | 0 | void set_last_full_compaction_schedule_time(int64_t millis) { |
279 | 0 | _last_full_compaction_schedule_millis = millis; |
280 | 0 | } |
281 | | |
282 | 0 | void set_last_single_compaction_failure_status(std::string status) { |
283 | 0 | _last_single_compaction_failure_status = std::move(status); |
284 | 0 | } |
285 | | |
286 | 0 | void set_last_fetched_version(Version version) { _last_fetched_version = std::move(version); } |
287 | | |
288 | | void delete_all_files(); |
289 | | |
290 | | void check_tablet_path_exists(); |
291 | | |
292 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_cumulative_compaction(); |
293 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction(); |
294 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction(); |
295 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_build_inverted_index( |
296 | | const std::set<int64_t>& alter_index_uids, bool is_drop_op); |
297 | | |
298 | | // used for single compaction to get the local versions |
299 | | // Single compaction does not require remote rowsets and cannot violate the cooldown semantics |
300 | | std::vector<Version> get_all_local_versions(); |
301 | | |
302 | | void calculate_cumulative_point(); |
303 | | // TODO(ygl): |
304 | 0 | bool is_primary_replica() { return false; } |
305 | | |
306 | | // return true if the checkpoint is actually done |
307 | | bool do_tablet_meta_checkpoint(); |
308 | | |
309 | | // Check whether the rowset is useful or not, unuseful rowset can be swept up then. |
310 | | // Rowset which is under tablet's management is useful, i.e. rowset is in |
311 | | // _rs_version_map, or _stale_rs_version_map. |
312 | | // Rowset whose version range is not covered by this tablet is also useful. |
313 | | bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta); |
314 | | |
315 | | void build_tablet_report_info(TTabletInfo* tablet_info, |
316 | | bool enable_consecutive_missing_check = false, |
317 | | bool enable_path_check = false); |
318 | | |
319 | | // return a json string to show the compaction status of this tablet |
320 | | void get_compaction_status(std::string* json_result); |
321 | | |
322 | | static Status prepare_compaction_and_calculate_permits( |
323 | | CompactionType compaction_type, const TabletSharedPtr& tablet, |
324 | | std::shared_ptr<CompactionMixin>& compaction, int64_t& permits); |
325 | | |
326 | | void execute_compaction(CompactionMixin& compaction); |
327 | | void execute_single_replica_compaction(SingleReplicaCompaction& compaction); |
328 | | |
329 | | void set_cumulative_compaction_policy( |
330 | 0 | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) { |
331 | 0 | _cumulative_compaction_policy = cumulative_compaction_policy; |
332 | 0 | } |
333 | | |
334 | 0 | std::shared_ptr<CumulativeCompactionPolicy> get_cumulative_compaction_policy() { |
335 | 0 | return _cumulative_compaction_policy; |
336 | 0 | } |
337 | | |
338 | 47.2k | void set_last_cumu_compaction_status(std::string status) { |
339 | 47.2k | _last_cumu_compaction_status = std::move(status); |
340 | 47.2k | } |
341 | | |
342 | 0 | std::string get_last_cumu_compaction_status() { return _last_cumu_compaction_status; } |
343 | | |
344 | 19.3k | void set_last_base_compaction_status(std::string status) { |
345 | 19.3k | _last_base_compaction_status = std::move(status); |
346 | 19.3k | } |
347 | | |
348 | 0 | std::string get_last_base_compaction_status() { return _last_base_compaction_status; } |
349 | | |
350 | 0 | void set_last_full_compaction_status(std::string status) { |
351 | 0 | _last_full_compaction_status = std::move(status); |
352 | 0 | } |
353 | | |
354 | 0 | std::string get_last_full_compaction_status() { return _last_full_compaction_status; } |
355 | | |
356 | | std::tuple<int64_t, int64_t> get_visible_version_and_time() const; |
357 | | |
358 | 472k | void set_visible_version(const std::shared_ptr<const VersionWithTime>& visible_version) { |
359 | 472k | std::atomic_store_explicit(&_visible_version, visible_version, std::memory_order_relaxed); |
360 | 472k | } |
361 | | |
362 | | bool should_fetch_from_peer(); |
363 | | |
364 | 0 | inline bool all_beta() const { |
365 | 0 | std::shared_lock rdlock(_meta_lock); |
366 | 0 | return _tablet_meta->all_beta(); |
367 | 0 | } |
368 | | |
369 | 0 | const TabletSchemaSPtr& tablet_schema_unlocked() const { return _max_version_schema; } |
370 | | |
371 | | Result<std::unique_ptr<RowsetWriter>> create_rowset_writer(RowsetWriterContext& context, |
372 | | bool vertical) override; |
373 | | |
374 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
375 | | const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info, |
376 | | int64_t txn_expiration = 0) override; |
377 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
378 | | RowsetWriterContext& context, const RowsetId& rowset_id); |
379 | | |
380 | | Status create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset); |
381 | | |
382 | | // MUST hold EXCLUSIVE `_meta_lock` |
383 | | void add_rowsets(const std::vector<RowsetSharedPtr>& to_add); |
384 | | // MUST hold EXCLUSIVE `_meta_lock` |
385 | | Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool move_to_stale); |
386 | | |
387 | | // MUST hold SHARED `_meta_lock` |
388 | 54.3k | const auto& rowset_map() const { return _rs_version_map; } |
389 | | // MUST hold SHARED `_meta_lock` |
390 | 54.3k | const auto& stale_rowset_map() const { return _stale_rs_version_map; } |
391 | | |
392 | | //////////////////////////////////////////////////////////////////////////// |
393 | | // begin cooldown functions |
394 | | //////////////////////////////////////////////////////////////////////////// |
395 | 1.01M | int64_t storage_policy_id() const { return _tablet_meta->storage_policy_id(); } |
396 | | void set_storage_policy_id(int64_t id) { _tablet_meta->set_storage_policy_id(id); } |
397 | | |
398 | 4.99M | int64_t last_failed_follow_cooldown_time() const { return _last_failed_follow_cooldown_time; } |
399 | | |
400 | | // Cooldown to remote fs. |
401 | | Status cooldown(RowsetSharedPtr rowset = nullptr); |
402 | | |
403 | | RowsetSharedPtr pick_cooldown_rowset(); |
404 | | |
405 | | RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size); |
406 | | |
407 | | struct CooldownConf { |
408 | | int64_t term = -1; |
409 | | int64_t cooldown_replica_id = -1; |
410 | | }; |
411 | | |
412 | 14 | CooldownConf cooldown_conf() const { |
413 | 14 | std::shared_lock rlock(_cooldown_conf_lock); |
414 | 14 | return _cooldown_conf; |
415 | 14 | } |
416 | | |
417 | 42 | CooldownConf cooldown_conf_unlocked() const { return _cooldown_conf; } |
418 | | |
419 | | // Return `true` if update success |
420 | | bool update_cooldown_conf(int64_t cooldown_term, int64_t cooldown_replica_id); |
421 | | |
422 | | Status remove_all_remote_rowsets(); |
423 | | |
424 | | void record_unused_remote_rowset(const RowsetId& rowset_id, const std::string& resource, |
425 | | int64_t num_segments); |
426 | | |
427 | | uint32_t calc_cold_data_compaction_score() const; |
428 | | |
429 | 19 | std::mutex& get_cold_compaction_lock() { return _cold_compaction_lock; } |
430 | | |
431 | 0 | std::shared_mutex& get_cooldown_conf_lock() { return _cooldown_conf_lock; } |
432 | | |
433 | | static void async_write_cooldown_meta(TabletSharedPtr tablet); |
434 | | // Return `ABORTED` if should not to retry again |
435 | | Status write_cooldown_meta(); |
436 | | //////////////////////////////////////////////////////////////////////////// |
437 | | // end cooldown functions |
438 | | //////////////////////////////////////////////////////////////////////////// |
439 | | |
440 | | CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override; |
441 | | Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, |
442 | | DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer, |
443 | | const RowsetIdUnorderedSet& cur_rowset_ids, int64_t lock_id = -1, |
444 | | int64_t next_visible_version = -1) override; |
445 | | |
446 | | void merge_delete_bitmap(const DeleteBitmap& delete_bitmap); |
447 | | bool check_all_rowset_segment(); |
448 | | |
449 | | void update_max_version_schema(const TabletSchemaSPtr& tablet_schema); |
450 | | |
451 | | void set_skip_compaction(bool skip, |
452 | | CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION, |
453 | | int64_t start = -1); |
454 | | bool should_skip_compaction(CompactionType compaction_type, int64_t now); |
455 | | |
456 | | std::vector<std::string> get_binlog_filepath(std::string_view binlog_version) const; |
457 | | std::pair<std::string, int64_t> get_binlog_info(std::string_view binlog_version) const; |
458 | | std::string get_rowset_binlog_meta(std::string_view binlog_version, |
459 | | std::string_view rowset_id) const; |
460 | | Status get_rowset_binlog_metas(const std::vector<int64_t>& binlog_versions, |
461 | | RowsetBinlogMetasPB* metas_pb); |
462 | | Status get_rowset_binlog_metas(Version binlog_versions, RowsetBinlogMetasPB* metas_pb); |
463 | | std::string get_segment_filepath(std::string_view rowset_id, |
464 | | std::string_view segment_index) const; |
465 | | std::string get_segment_filepath(std::string_view rowset_id, int64_t segment_index) const; |
466 | | bool can_add_binlog(uint64_t total_binlog_size) const; |
467 | | void gc_binlogs(int64_t version); |
468 | | Status ingest_binlog_metas(RowsetBinlogMetasPB* metas_pb); |
469 | | |
470 | | void report_error(const Status& st); |
471 | | |
472 | 0 | inline int64_t get_io_error_times() const { return _io_error_times; } |
473 | | |
474 | 1.60M | inline bool is_io_error_too_times() const { |
475 | 1.60M | return config::max_tablet_io_errors > 0 && _io_error_times >= config::max_tablet_io_errors; |
476 | 1.60M | } |
477 | | |
478 | 382 | int64_t get_table_id() { return _tablet_meta->table_id(); } |
479 | | |
480 | | // binlog related functions |
481 | | bool is_enable_binlog(); |
482 | 0 | bool is_binlog_enabled() { return _tablet_meta->binlog_config().is_enable(); } |
483 | 0 | int64_t binlog_ttl_ms() const { return _tablet_meta->binlog_config().ttl_seconds(); } |
484 | 0 | int64_t binlog_max_bytes() const { return _tablet_meta->binlog_config().max_bytes(); } |
485 | | |
486 | | void set_binlog_config(BinlogConfig binlog_config); |
487 | | |
488 | 0 | void set_is_full_compaction_running(bool is_full_compaction_running) { |
489 | 0 | _is_full_compaction_running = is_full_compaction_running; |
490 | 0 | } |
491 | 0 | inline bool is_full_compaction_running() const { return _is_full_compaction_running; } |
492 | | void clear_cache() override; |
493 | | |
494 | | int32_t get_compaction_score() const { return _compaction_score; } |
495 | | |
496 | 0 | void set_compaction_score(int32_t compaction_score) { _compaction_score = compaction_score; } |
497 | | |
498 | 12.4k | void add_compaction_score(int32_t score) { |
499 | 12.4k | if (_compaction_score < 0) { |
500 | 12.0k | return; |
501 | 12.0k | } |
502 | 462 | _compaction_score += score; |
503 | 462 | } |
504 | | |
505 | 0 | void minus_compaction_score(int32_t score) { |
506 | 0 | if (_compaction_score < 0) { |
507 | 0 | return; |
508 | 0 | } |
509 | 0 | _compaction_score -= score; |
510 | 0 | } |
511 | | |
512 | | private: |
513 | | Status _init_once_action(); |
514 | | bool _contains_rowset(const RowsetId rowset_id); |
515 | | Status _contains_version(const Version& version); |
516 | | |
517 | | // Returns: |
518 | | // version: the max continuous version from beginning |
519 | | // max_version: the max version of this tablet |
520 | | void _max_continuous_version_from_beginning_unlocked(Version* version, Version* max_version, |
521 | | bool* has_version_cross) const; |
522 | | RowsetSharedPtr _rowset_with_largest_size(); |
523 | | /// Delete stale rowset by version. This method not only delete the version in expired rowset map, |
524 | | /// but also delete the version in rowset meta vector. |
525 | | void _delete_stale_rowset_by_version(const Version& version); |
526 | | |
527 | | uint32_t _calc_cumulative_compaction_score( |
528 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
529 | | uint32_t _calc_base_compaction_score() const; |
530 | | |
531 | | std::vector<RowsetSharedPtr> _pick_visible_rowsets_to_compaction(int64_t min_start_version, |
532 | | int64_t max_start_version); |
533 | | |
534 | | void _init_context_common_fields(RowsetWriterContext& context); |
535 | | |
536 | | //////////////////////////////////////////////////////////////////////////// |
537 | | // begin cooldown functions |
538 | | //////////////////////////////////////////////////////////////////////////// |
539 | | Status _cooldown_data(RowsetSharedPtr rowset); |
540 | | Status _follow_cooldowned_data(); |
541 | | Status _read_cooldown_meta(const StorageResource& storage_resource, |
542 | | TabletMetaPB* tablet_meta_pb); |
543 | | bool _has_data_to_cooldown(); |
544 | | int64_t _get_newest_cooldown_time(const RowsetSharedPtr& rowset); |
545 | | //////////////////////////////////////////////////////////////////////////// |
546 | | // end cooldown functions |
547 | | //////////////////////////////////////////////////////////////////////////// |
548 | | |
549 | | void _clear_cache_by_rowset(const BetaRowsetSharedPtr& rowset); |
550 | | void check_table_size_correctness(); |
551 | | std::string get_segment_path(const RowsetMetaSharedPtr& rs_meta, int64_t seg_id); |
552 | | int64_t get_segment_file_size(const RowsetMetaSharedPtr& rs_meta); |
553 | | int64_t get_inverted_index_file_size(const RowsetMetaSharedPtr& rs_meta); |
554 | | |
555 | | public: |
556 | | static const int64_t K_INVALID_CUMULATIVE_POINT = -1; |
557 | | |
558 | | private: |
559 | | StorageEngine& _engine; |
560 | | DataDir* _data_dir = nullptr; |
561 | | |
562 | | std::string _tablet_path; |
563 | | |
564 | | DorisCallOnce<Status> _init_once; |
565 | | // meta store lock is used for prevent 2 threads do checkpoint concurrently |
566 | | // it will be used in econ-mode in the future |
567 | | std::shared_mutex _meta_store_lock; |
568 | | std::mutex _ingest_lock; |
569 | | std::mutex _base_compaction_lock; |
570 | | std::mutex _cumulative_compaction_lock; |
571 | | std::shared_timed_mutex _migration_lock; |
572 | | std::mutex _build_inverted_index_lock; |
573 | | |
574 | | // In unique key table with MoW, we should guarantee that only one |
575 | | // writer can update rowset and delete bitmap at the same time. |
576 | | // We use a separate lock rather than _meta_lock, to avoid blocking read queries |
577 | | // during publish_txn, which might take hundreds of milliseconds |
578 | | mutable std::mutex _rowset_update_lock; |
579 | | |
580 | | // if this tablet is broken, set to true. default is false |
581 | | std::atomic<bool> _is_bad; |
582 | | // timestamp of last cumu compaction failure |
583 | | std::atomic<int64_t> _last_cumu_compaction_failure_millis; |
584 | | // timestamp of last base compaction failure |
585 | | std::atomic<int64_t> _last_base_compaction_failure_millis; |
586 | | // timestamp of last full compaction failure |
587 | | std::atomic<int64_t> _last_full_compaction_failure_millis; |
588 | | // timestamp of last cumu compaction success |
589 | | std::atomic<int64_t> _last_cumu_compaction_success_millis; |
590 | | // timestamp of last base compaction success |
591 | | std::atomic<int64_t> _last_base_compaction_success_millis; |
592 | | // timestamp of last full compaction success |
593 | | std::atomic<int64_t> _last_full_compaction_success_millis; |
594 | | // timestamp of last cumu compaction schedule time |
595 | | std::atomic<int64_t> _last_cumu_compaction_schedule_millis; |
596 | | // timestamp of last base compaction schedule time |
597 | | std::atomic<int64_t> _last_base_compaction_schedule_millis; |
598 | | // timestamp of last full compaction schedule time |
599 | | std::atomic<int64_t> _last_full_compaction_schedule_millis; |
600 | | std::atomic<int64_t> _cumulative_point; |
601 | | std::atomic<int64_t> _cumulative_promotion_size; |
602 | | std::atomic<int32_t> _newly_created_rowset_num; |
603 | | std::atomic<int64_t> _last_checkpoint_time; |
604 | | std::string _last_cumu_compaction_status; |
605 | | std::string _last_base_compaction_status; |
606 | | std::string _last_full_compaction_status; |
607 | | |
608 | | // single replica compaction status |
609 | | std::string _last_single_compaction_failure_status; |
610 | | Version _last_fetched_version; |
611 | | |
612 | | // cumulative compaction policy |
613 | | std::shared_ptr<CumulativeCompactionPolicy> _cumulative_compaction_policy; |
614 | | std::string_view _cumulative_compaction_type; |
615 | | |
616 | | // use a separate thread to check all tablets paths existence |
617 | | std::atomic<bool> _is_tablet_path_exists; |
618 | | |
619 | | int64_t _last_missed_version; |
620 | | int64_t _last_missed_time_s; |
621 | | |
622 | | bool _skip_cumu_compaction = false; |
623 | | int64_t _skip_cumu_compaction_ts; |
624 | | |
625 | | bool _skip_base_compaction = false; |
626 | | int64_t _skip_base_compaction_ts; |
627 | | |
628 | | // cooldown related |
629 | | CooldownConf _cooldown_conf; |
630 | | // `_cooldown_conf_lock` is used to serialize update cooldown conf and all operations that: |
631 | | // 1. read cooldown conf |
632 | | // 2. upload rowsets to remote storage |
633 | | // 3. update cooldown meta id |
634 | | mutable std::shared_mutex _cooldown_conf_lock; |
635 | | // `_cold_compaction_lock` is used to serialize cold data compaction and all operations that |
636 | | // may delete compaction input rowsets. |
637 | | std::mutex _cold_compaction_lock; |
638 | | int64_t _last_failed_follow_cooldown_time = 0; |
639 | | |
640 | | int64_t _io_error_times = 0; |
641 | | |
642 | | // partition's visible version. it sync from fe, but not real-time. |
643 | | std::shared_ptr<const VersionWithTime> _visible_version; |
644 | | |
645 | | std::atomic_bool _is_full_compaction_running = false; |
646 | | |
647 | | int32_t _compaction_score = -1; |
648 | | int32_t _score_check_cnt = 0; |
649 | | }; |
650 | | |
651 | 41.9k | inline CumulativeCompactionPolicy* Tablet::cumulative_compaction_policy() { |
652 | 41.9k | return _cumulative_compaction_policy.get(); |
653 | 41.9k | } |
654 | | |
655 | 18.6M | inline bool Tablet::init_succeeded() { |
656 | 18.6M | return _init_once.has_called() && _init_once.stored_result().ok(); |
657 | 18.6M | } |
658 | | |
659 | 21.7M | inline bool Tablet::is_used() { |
660 | 21.7M | return !_is_bad && _data_dir->is_used(); |
661 | 21.7M | } |
662 | | |
663 | 458k | inline void Tablet::register_tablet_into_dir() { |
664 | 458k | _data_dir->register_tablet(this); |
665 | 458k | } |
666 | | |
667 | 13.6k | inline void Tablet::deregister_tablet_from_dir() { |
668 | 13.6k | _data_dir->deregister_tablet(this); |
669 | 13.6k | } |
670 | | |
671 | 632k | inline int64_t Tablet::cumulative_layer_point() const { |
672 | 632k | return _cumulative_point; |
673 | 632k | } |
674 | | |
675 | 2.27k | inline void Tablet::set_cumulative_layer_point(int64_t new_point) { |
676 | | // cumulative point should only be reset to -1, or be increased |
677 | 2.27k | CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) |
678 | 0 | << "Unexpected cumulative point: " << new_point |
679 | 0 | << ", origin: " << _cumulative_point.load(); |
680 | 2.27k | _cumulative_point = new_point; |
681 | 2.27k | } |
682 | | |
683 | 41.8k | inline int64_t Tablet::cumulative_promotion_size() const { |
684 | 41.8k | return _cumulative_promotion_size; |
685 | 41.8k | } |
686 | | |
687 | 71.7k | inline void Tablet::set_cumulative_promotion_size(int64_t new_size) { |
688 | 71.7k | _cumulative_promotion_size = new_size; |
689 | 71.7k | } |
690 | | |
691 | | // TODO(lingbin): Why other methods that need to get information from _tablet_meta |
692 | | // are not locked, here needs a comment to explain. |
693 | 1.79k | inline size_t Tablet::tablet_footprint() { |
694 | 1.79k | std::shared_lock rdlock(_meta_lock); |
695 | 1.79k | return _tablet_meta->tablet_footprint(); |
696 | 1.79k | } |
697 | | |
698 | 2.27M | inline size_t Tablet::tablet_local_size() { |
699 | 2.27M | std::shared_lock rdlock(_meta_lock); |
700 | 2.27M | return _tablet_meta->tablet_local_size(); |
701 | 2.27M | } |
702 | | |
703 | 2.27M | inline size_t Tablet::tablet_remote_size() { |
704 | 2.27M | std::shared_lock rdlock(_meta_lock); |
705 | 2.27M | return _tablet_meta->tablet_remote_size(); |
706 | 2.27M | } |
707 | | |
708 | | // TODO(lingbin): Why other methods which need to get information from _tablet_meta |
709 | | // are not locked, here needs a comment to explain. |
710 | | inline size_t Tablet::num_rows() { |
711 | | std::shared_lock rdlock(_meta_lock); |
712 | | return _tablet_meta->num_rows(); |
713 | | } |
714 | | |
715 | 5.90M | inline size_t Tablet::version_count() const { |
716 | 5.90M | std::shared_lock rdlock(_meta_lock); |
717 | 5.90M | return _tablet_meta->version_count(); |
718 | 5.90M | } |
719 | | |
720 | 410 | inline size_t Tablet::stale_version_count() const { |
721 | 410 | std::shared_lock rdlock(_meta_lock); |
722 | 410 | return _tablet_meta->stale_version_count(); |
723 | 410 | } |
724 | | |
725 | 500k | inline Version Tablet::max_version() const { |
726 | 500k | std::shared_lock rdlock(_meta_lock); |
727 | 500k | return _tablet_meta->max_version(); |
728 | 500k | } |
729 | | |
730 | 5.90M | inline uint64_t Tablet::segment_count() const { |
731 | 5.90M | std::shared_lock rdlock(_meta_lock); |
732 | 5.90M | uint64_t segment_nums = 0; |
733 | 11.0M | for (const auto& rs_meta : _tablet_meta->all_rs_metas()) { |
734 | 11.0M | segment_nums += rs_meta->num_segments(); |
735 | 11.0M | } |
736 | 5.90M | return segment_nums; |
737 | 5.90M | } |
738 | | |
739 | 0 | inline SortType Tablet::sort_type() const { |
740 | 0 | return _tablet_meta->tablet_schema()->sort_type(); |
741 | 0 | } |
742 | | |
743 | 0 | inline size_t Tablet::sort_col_num() const { |
744 | 0 | return _tablet_meta->tablet_schema()->sort_col_num(); |
745 | 0 | } |
746 | | |
747 | 0 | inline size_t Tablet::num_columns() const { |
748 | 0 | return _tablet_meta->tablet_schema()->num_columns(); |
749 | 0 | } |
750 | | |
751 | 0 | inline size_t Tablet::num_null_columns() const { |
752 | 0 | return _tablet_meta->tablet_schema()->num_null_columns(); |
753 | 0 | } |
754 | | |
755 | 0 | inline size_t Tablet::num_short_key_columns() const { |
756 | 0 | return _tablet_meta->tablet_schema()->num_short_key_columns(); |
757 | 0 | } |
758 | | |
759 | 0 | inline size_t Tablet::num_rows_per_row_block() const { |
760 | 0 | return _tablet_meta->tablet_schema()->num_rows_per_row_block(); |
761 | 0 | } |
762 | | |
763 | 0 | inline double Tablet::bloom_filter_fpp() const { |
764 | 0 | return _tablet_meta->tablet_schema()->bloom_filter_fpp(); |
765 | 0 | } |
766 | | |
767 | 0 | inline size_t Tablet::next_unique_id() const { |
768 | 0 | return _tablet_meta->tablet_schema()->next_column_unique_id(); |
769 | 0 | } |
770 | | |
771 | 410 | inline int64_t Tablet::avg_rs_meta_serialize_size() const { |
772 | 410 | return _tablet_meta->avg_rs_meta_serialize_size(); |
773 | 410 | } |
774 | | |
775 | | #include "common/compile_check_end.h" |
776 | | } // namespace doris |