/root/doris/be/src/olap/tablet.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <butil/macros.h> |
21 | | #include <glog/logging.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <cstddef> |
25 | | #include <cstdint> |
26 | | #include <functional> |
27 | | #include <memory> |
28 | | #include <mutex> |
29 | | #include <ostream> |
30 | | #include <set> |
31 | | #include <shared_mutex> |
32 | | #include <string> |
33 | | #include <string_view> |
34 | | #include <utility> |
35 | | #include <vector> |
36 | | |
37 | | #include "common/config.h" |
38 | | #include "common/status.h" |
39 | | #include "olap/base_tablet.h" |
40 | | #include "olap/binlog_config.h" |
41 | | #include "olap/data_dir.h" |
42 | | #include "olap/olap_common.h" |
43 | | #include "olap/partial_update_info.h" |
44 | | #include "olap/rowset/rowset.h" |
45 | | #include "olap/rowset/rowset_meta.h" |
46 | | #include "olap/rowset/rowset_reader.h" |
47 | | #include "olap/rowset/segment_v2/segment.h" |
48 | | #include "olap/version_graph.h" |
49 | | #include "segment_loader.h" |
50 | | #include "util/metrics.h" |
51 | | #include "util/once.h" |
52 | | #include "util/slice.h" |
53 | | |
54 | | namespace bvar { |
55 | | template <typename T> |
56 | | class Adder; |
57 | | } |
58 | | |
59 | | namespace doris { |
60 | | |
61 | | class Tablet; |
62 | | class CumulativeCompactionPolicy; |
63 | | class CompactionMixin; |
64 | | class SingleReplicaCompaction; |
65 | | class RowsetWriter; |
66 | | struct RowsetWriterContext; |
67 | | class TTabletInfo; |
68 | | class TabletMetaPB; |
69 | | class TupleDescriptor; |
70 | | class CalcDeleteBitmapToken; |
71 | | enum CompressKind : int; |
72 | | class RowsetBinlogMetasPB; |
73 | | |
74 | | namespace io { |
75 | | class RemoteFileSystem; |
76 | | } // namespace io |
77 | | namespace vectorized { |
78 | | class Block; |
79 | | } // namespace vectorized |
80 | | struct RowLocation; |
81 | | enum KeysType : int; |
82 | | enum SortType : int; |
83 | | |
84 | | enum TabletStorageType { STORAGE_TYPE_LOCAL, STORAGE_TYPE_REMOTE, STORAGE_TYPE_REMOTE_AND_LOCAL }; |
85 | | |
86 | | extern bvar::Adder<uint64_t> unused_remote_rowset_num; |
87 | | |
88 | | static inline constexpr auto TRACE_TABLET_LOCK_THRESHOLD = std::chrono::seconds(1); |
89 | | |
90 | | struct WriteCooldownMetaExecutors { |
91 | | WriteCooldownMetaExecutors(size_t executor_nums = 5); |
92 | | |
93 | | void stop(); |
94 | | |
95 | | void submit(TabletSharedPtr tablet); |
96 | 5 | size_t _get_executor_pos(int64_t tablet_id) const { |
97 | 5 | return std::hash<int64_t>()(tablet_id) % _executor_nums; |
98 | 5 | }; |
99 | | // Each executor is a mpsc to ensure uploads of the same tablet meta are not concurrent |
100 | | // FIXME(AlexYue): Use mpsc instead of `ThreadPool` with 1 thread |
101 | | // We use PriorityThreadPool since it would call status inside it's `shutdown` function. |
102 | | // Consider one situation where the StackTraceCache's singleton is detructed before |
103 | | // this WriteCooldownMetaExecutors's singleton, then invoking the status would also call |
104 | | // StackTraceCache which would then result in heap use after free like #23834 |
105 | | std::vector<std::unique_ptr<PriorityThreadPool>> _executors; |
106 | | std::unordered_set<int64_t> _pending_tablets; |
107 | | std::mutex _latch; |
108 | | size_t _executor_nums; |
109 | | }; |
110 | | |
111 | | class Tablet final : public BaseTablet { |
112 | | public: |
113 | | Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* data_dir, |
114 | | const std::string_view& cumulative_compaction_type = ""); |
115 | | |
116 | 3.76k | DataDir* data_dir() const { return _data_dir; } |
117 | 268 | int64_t replica_id() const { return _tablet_meta->replica_id(); } |
118 | | |
119 | 11.8k | const std::string& tablet_path() const { return _tablet_path; } |
120 | | |
121 | | bool set_tablet_schema_into_rowset_meta(); |
122 | | Status init(); |
123 | | bool init_succeeded(); |
124 | | |
125 | | bool is_used(); |
126 | | |
127 | | void register_tablet_into_dir(); |
128 | | void deregister_tablet_from_dir(); |
129 | | |
130 | | void save_meta(); |
131 | | // Used in clone task, to update local meta when finishing a clone job |
132 | | Status revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add, |
133 | | const std::vector<RowsetSharedPtr>& to_delete, |
134 | | bool is_incremental_clone); |
135 | | |
136 | | int64_t cumulative_layer_point() const; |
137 | | void set_cumulative_layer_point(int64_t new_point); |
138 | | inline int64_t cumulative_promotion_size() const; |
139 | | inline void set_cumulative_promotion_size(int64_t new_size); |
140 | | |
141 | | // Disk space occupied by tablet, contain local and remote. |
142 | | size_t tablet_footprint() override; |
143 | | // Local disk space occupied by tablet. |
144 | | size_t tablet_local_size(); |
145 | | // Remote disk space occupied by tablet. |
146 | | size_t tablet_remote_size(); |
147 | | |
148 | | size_t num_rows(); |
149 | | int version_count() const; |
150 | | int stale_version_count() const; |
151 | | bool exceed_version_limit(int32_t limit) override; |
152 | | uint64_t segment_count() const; |
153 | | Version max_version() const; |
154 | | CumulativeCompactionPolicy* cumulative_compaction_policy(); |
155 | | |
156 | | // properties encapsulated in TabletSchema |
157 | | SortType sort_type() const; |
158 | | size_t sort_col_num() const; |
159 | | size_t num_columns() const; |
160 | | size_t num_null_columns() const; |
161 | | size_t num_short_key_columns() const; |
162 | | size_t num_rows_per_row_block() const; |
163 | | CompressKind compress_kind() const; |
164 | | double bloom_filter_fpp() const; |
165 | | size_t next_unique_id() const; |
166 | | size_t row_size() const; |
167 | | int64_t avg_rs_meta_serialize_size() const; |
168 | | |
169 | | // operation in rowsets |
170 | | Status add_rowset(RowsetSharedPtr rowset); |
171 | | Status create_initial_rowset(const int64_t version); |
172 | | |
173 | | // MUST hold EXCLUSIVE `_meta_lock`. |
174 | | Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add, |
175 | | std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false); |
176 | | bool rowset_exists_unlocked(const RowsetSharedPtr& rowset); |
177 | | |
178 | | Status add_inc_rowset(const RowsetSharedPtr& rowset); |
179 | | /// Delete stale rowset by timing. This delete policy uses now() minutes |
180 | | /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset |
181 | | /// to delete. When rowset is deleted, it will be added to StorageEngine unused map and record |
182 | | /// need to delete flag. |
183 | | void delete_expired_stale_rowset(); |
184 | | |
185 | | // Given spec_version, find a continuous version path and store it in version_path. |
186 | | // If quiet is true, then only "does this path exist" is returned. |
187 | | // If skip_missing_version is true, return ok even there are missing versions. |
188 | | Status capture_consistent_versions_unlocked(const Version& spec_version, Versions* version_path, |
189 | | bool skip_missing_version, bool quiet) const; |
190 | | |
191 | | // if quiet is true, no error log will be printed if there are missing versions |
192 | | Status check_version_integrity(const Version& version, bool quiet = false); |
193 | | bool check_version_exist(const Version& version) const; |
194 | | void acquire_version_and_rowsets( |
195 | | std::vector<std::pair<Version, RowsetSharedPtr>>* version_rowsets) const; |
196 | | |
197 | | Status capture_consistent_rowsets_unlocked( |
198 | | const Version& spec_version, std::vector<RowsetSharedPtr>* rowsets) const override; |
199 | | |
200 | | // If skip_missing_version is true, skip versions if they are missing. |
201 | | Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits, |
202 | | bool skip_missing_version) override; |
203 | | |
204 | | // Find the missed versions until the spec_version. |
205 | | // |
206 | | // for example: |
207 | | // [0-4][5-5][8-8][9-9][14-14] |
208 | | // if spec_version = 12, it will return [6, 6], [7, 7], [10, 10], [11, 11], [12, 12] |
209 | | Versions calc_missed_versions(int64_t spec_version, Versions existing_versions) const override; |
210 | | |
211 | | // meta lock |
212 | 764 | std::shared_mutex& get_header_lock() { return _meta_lock; } |
213 | 2 | std::mutex& get_rowset_update_lock() { return _rowset_update_lock; } |
214 | 31 | std::mutex& get_push_lock() { return _ingest_lock; } |
215 | 20 | std::mutex& get_base_compaction_lock() { return _base_compaction_lock; } |
216 | 287 | std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; } |
217 | 1 | std::shared_mutex& get_meta_store_lock() { return _meta_store_lock; } |
218 | | |
219 | 50 | std::shared_timed_mutex& get_migration_lock() { return _migration_lock; } |
220 | | |
221 | 18 | std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; } |
222 | | |
223 | | // operation for compaction |
224 | | bool can_do_compaction(size_t path_hash, CompactionType compaction_type); |
225 | | bool suitable_for_compaction( |
226 | | CompactionType compaction_type, |
227 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
228 | | |
229 | | uint32_t calc_compaction_score(); |
230 | | |
231 | | // This function to find max continuous version from the beginning. |
232 | | // For example: If there are 1, 2, 3, 5, 6, 7 versions belongs tablet, then 3 is target. |
233 | | // 3 will be saved in "version", and 7 will be saved in "max_version", if max_version != nullptr |
234 | | void max_continuous_version_from_beginning(Version* version, Version* max_version = nullptr); |
235 | | |
236 | 0 | void set_bad(bool is_bad) { _is_bad = is_bad; } |
237 | | |
238 | 257 | int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; } |
239 | 0 | void set_last_cumu_compaction_failure_time(int64_t millis) { |
240 | 0 | _last_cumu_compaction_failure_millis = millis; |
241 | 0 | } |
242 | | |
243 | 0 | int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; } |
244 | 0 | void set_last_base_compaction_failure_time(int64_t millis) { |
245 | 0 | _last_base_compaction_failure_millis = millis; |
246 | 0 | } |
247 | | |
248 | 0 | int64_t last_full_compaction_failure_time() { return _last_full_compaction_failure_millis; } |
249 | 0 | void set_last_full_compaction_failure_time(int64_t millis) { |
250 | 0 | _last_full_compaction_failure_millis = millis; |
251 | 0 | } |
252 | | |
253 | 7 | int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; } |
254 | 3 | void set_last_cumu_compaction_success_time(int64_t millis) { |
255 | 3 | _last_cumu_compaction_success_millis = millis; |
256 | 3 | } |
257 | | |
258 | 0 | int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; } |
259 | 0 | void set_last_base_compaction_success_time(int64_t millis) { |
260 | 0 | _last_base_compaction_success_millis = millis; |
261 | 0 | } |
262 | | |
263 | 0 | int64_t last_full_compaction_success_time() { return _last_full_compaction_success_millis; } |
264 | 0 | void set_last_full_compaction_success_time(int64_t millis) { |
265 | 0 | _last_full_compaction_success_millis = millis; |
266 | 0 | } |
267 | | |
268 | 0 | int64_t last_base_compaction_schedule_time() { return _last_base_compaction_schedule_millis; } |
269 | 0 | void set_last_base_compaction_schedule_time(int64_t millis) { |
270 | 0 | _last_base_compaction_schedule_millis = millis; |
271 | 0 | } |
272 | | |
273 | 0 | void set_last_single_compaction_failure_status(std::string status) { |
274 | 0 | _last_single_compaction_failure_status = std::move(status); |
275 | 0 | } |
276 | | |
277 | 0 | void set_last_fetched_version(Version version) { _last_fetched_version = std::move(version); } |
278 | | |
279 | | void delete_all_files(); |
280 | | |
281 | | void check_tablet_path_exists(); |
282 | | |
283 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_cumulative_compaction(); |
284 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction(); |
285 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction(); |
286 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_build_inverted_index( |
287 | | const std::set<int64_t>& alter_index_uids, bool is_drop_op); |
288 | | |
289 | | // used for single compaction to get the local versions |
290 | | // Single compaction does not require remote rowsets and cannot violate the cooldown semantics |
291 | | std::vector<Version> get_all_local_versions(); |
292 | | |
293 | | void calculate_cumulative_point(); |
294 | | // TODO(ygl): |
295 | 0 | bool is_primary_replica() { return false; } |
296 | | |
297 | | // return true if the checkpoint is actually done |
298 | | bool do_tablet_meta_checkpoint(); |
299 | | |
300 | | // Check whether the rowset is useful or not, unuseful rowset can be swept up then. |
301 | | // Rowset which is under tablet's management is useful, i.e. rowset is in |
302 | | // _rs_version_map, or _stale_rs_version_map. |
303 | | // Rowset whose version range is not covered by this tablet is also useful. |
304 | | bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta); |
305 | | |
306 | | void build_tablet_report_info(TTabletInfo* tablet_info, |
307 | | bool enable_consecutive_missing_check = false, |
308 | | bool enable_path_check = false); |
309 | | |
310 | | // return a json string to show the compaction status of this tablet |
311 | | void get_compaction_status(std::string* json_result); |
312 | | |
313 | | static Status prepare_compaction_and_calculate_permits( |
314 | | CompactionType compaction_type, const TabletSharedPtr& tablet, |
315 | | std::shared_ptr<CompactionMixin>& compaction, int64_t& permits); |
316 | | |
317 | | void execute_compaction(CompactionMixin& compaction); |
318 | | void execute_single_replica_compaction(SingleReplicaCompaction& compaction); |
319 | | |
320 | | void set_cumulative_compaction_policy( |
321 | 0 | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) { |
322 | 0 | _cumulative_compaction_policy = cumulative_compaction_policy; |
323 | 0 | } |
324 | | |
325 | 0 | std::shared_ptr<CumulativeCompactionPolicy> get_cumulative_compaction_policy() { |
326 | 0 | return _cumulative_compaction_policy; |
327 | 0 | } |
328 | | |
329 | 0 | void set_last_base_compaction_status(std::string status) { |
330 | 0 | _last_base_compaction_status = std::move(status); |
331 | 0 | } |
332 | | |
333 | 0 | std::string get_last_base_compaction_status() { return _last_base_compaction_status; } |
334 | | |
335 | | std::tuple<int64_t, int64_t> get_visible_version_and_time() const; |
336 | | |
337 | 558 | void set_visible_version(const std::shared_ptr<const VersionWithTime>& visible_version) { |
338 | 558 | std::atomic_store_explicit(&_visible_version, visible_version, std::memory_order_relaxed); |
339 | 558 | } |
340 | | |
341 | | bool should_fetch_from_peer(); |
342 | | |
343 | 0 | inline bool all_beta() const { |
344 | 0 | std::shared_lock rdlock(_meta_lock); |
345 | 0 | return _tablet_meta->all_beta(); |
346 | 0 | } |
347 | | |
348 | 0 | const TabletSchemaSPtr& tablet_schema_unlocked() const { return _max_version_schema; } |
349 | | |
350 | | Result<std::unique_ptr<RowsetWriter>> create_rowset_writer(RowsetWriterContext& context, |
351 | | bool vertical) override; |
352 | | |
353 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
354 | | const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info, |
355 | | int64_t txn_expiration = 0) override; |
356 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
357 | | RowsetWriterContext& context, const RowsetId& rowset_id); |
358 | | |
359 | | Status create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset); |
360 | | |
361 | | // MUST hold EXCLUSIVE `_meta_lock` |
362 | | void add_rowsets(const std::vector<RowsetSharedPtr>& to_add); |
363 | | // MUST hold EXCLUSIVE `_meta_lock` |
364 | | Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool move_to_stale); |
365 | | |
366 | | // MUST hold SHARED `_meta_lock` |
367 | 485 | const auto& rowset_map() const { return _rs_version_map; } |
368 | | // MUST hold SHARED `_meta_lock` |
369 | 480 | const auto& stale_rowset_map() const { return _stale_rs_version_map; } |
370 | | |
371 | | //////////////////////////////////////////////////////////////////////////// |
372 | | // begin cooldown functions |
373 | | //////////////////////////////////////////////////////////////////////////// |
374 | 22 | int64_t storage_policy_id() const { return _tablet_meta->storage_policy_id(); } |
375 | 4 | void set_storage_policy_id(int64_t id) { _tablet_meta->set_storage_policy_id(id); } |
376 | | |
377 | 0 | int64_t last_failed_follow_cooldown_time() const { return _last_failed_follow_cooldown_time; } |
378 | | |
379 | | // Cooldown to remote fs. |
380 | | Status cooldown(RowsetSharedPtr rowset = nullptr); |
381 | | |
382 | | RowsetSharedPtr pick_cooldown_rowset(); |
383 | | |
384 | | RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size); |
385 | | |
386 | | struct CooldownConf { |
387 | | int64_t term = -1; |
388 | | int64_t cooldown_replica_id = -1; |
389 | | }; |
390 | | |
391 | 0 | CooldownConf cooldown_conf() const { |
392 | 0 | std::shared_lock rlock(_cooldown_conf_lock); |
393 | 0 | return _cooldown_conf; |
394 | 0 | } |
395 | | |
396 | 0 | CooldownConf cooldown_conf_unlocked() const { return _cooldown_conf; } |
397 | | |
398 | | // Return `true` if update success |
399 | | bool update_cooldown_conf(int64_t cooldown_term, int64_t cooldown_replica_id); |
400 | | |
401 | | Status remove_all_remote_rowsets(); |
402 | | |
403 | | void record_unused_remote_rowset(const RowsetId& rowset_id, const std::string& resource, |
404 | | int64_t num_segments); |
405 | | |
406 | | uint32_t calc_cold_data_compaction_score() const; |
407 | | |
408 | 18 | std::mutex& get_cold_compaction_lock() { return _cold_compaction_lock; } |
409 | | |
410 | 0 | std::shared_mutex& get_cooldown_conf_lock() { return _cooldown_conf_lock; } |
411 | | |
412 | | static void async_write_cooldown_meta(TabletSharedPtr tablet); |
413 | | // Return `ABORTED` if should not to retry again |
414 | | Status write_cooldown_meta(); |
415 | | //////////////////////////////////////////////////////////////////////////// |
416 | | // end cooldown functions |
417 | | //////////////////////////////////////////////////////////////////////////// |
418 | | |
419 | | CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override; |
420 | | Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, |
421 | | DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer, |
422 | | const RowsetIdUnorderedSet& cur_rowset_ids, |
423 | | int64_t lock_id = -1) override; |
424 | | |
425 | | void merge_delete_bitmap(const DeleteBitmap& delete_bitmap); |
426 | | bool check_all_rowset_segment(); |
427 | | |
428 | | void update_max_version_schema(const TabletSchemaSPtr& tablet_schema); |
429 | | |
430 | | void set_skip_compaction(bool skip, |
431 | | CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION, |
432 | | int64_t start = -1); |
433 | | bool should_skip_compaction(CompactionType compaction_type, int64_t now); |
434 | | |
435 | | std::vector<std::string> get_binlog_filepath(std::string_view binlog_version) const; |
436 | | std::pair<std::string, int64_t> get_binlog_info(std::string_view binlog_version) const; |
437 | | std::string get_rowset_binlog_meta(std::string_view binlog_version, |
438 | | std::string_view rowset_id) const; |
439 | | Status get_rowset_binlog_metas(const std::vector<int64_t>& binlog_versions, |
440 | | RowsetBinlogMetasPB* metas_pb); |
441 | | Status get_rowset_binlog_metas(Version binlog_versions, RowsetBinlogMetasPB* metas_pb); |
442 | | std::string get_segment_filepath(std::string_view rowset_id, |
443 | | std::string_view segment_index) const; |
444 | | std::string get_segment_filepath(std::string_view rowset_id, int64_t segment_index) const; |
445 | | bool can_add_binlog(uint64_t total_binlog_size) const; |
446 | | void gc_binlogs(int64_t version); |
447 | | Status ingest_binlog_metas(RowsetBinlogMetasPB* metas_pb); |
448 | | |
449 | | void report_error(const Status& st); |
450 | | |
451 | 0 | inline int64_t get_io_error_times() const { return _io_error_times; } |
452 | | |
453 | 0 | inline bool is_io_error_too_times() const { |
454 | 0 | return config::max_tablet_io_errors > 0 && _io_error_times >= config::max_tablet_io_errors; |
455 | 0 | } |
456 | | |
457 | 0 | int64_t get_table_id() { return _tablet_meta->table_id(); } |
458 | | |
459 | | // binlog related functions |
460 | | bool is_enable_binlog(); |
461 | 0 | bool is_binlog_enabled() { return _tablet_meta->binlog_config().is_enable(); } |
462 | 0 | int64_t binlog_ttl_ms() const { return _tablet_meta->binlog_config().ttl_seconds(); } |
463 | 0 | int64_t binlog_max_bytes() const { return _tablet_meta->binlog_config().max_bytes(); } |
464 | | |
465 | | void set_binlog_config(BinlogConfig binlog_config); |
466 | | |
467 | 0 | void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; } |
468 | 0 | bool is_alter_failed() { return _alter_failed; } |
469 | | |
470 | 0 | void set_is_full_compaction_running(bool is_full_compaction_running) { |
471 | 0 | _is_full_compaction_running = is_full_compaction_running; |
472 | 0 | } |
473 | 0 | inline bool is_full_compaction_running() const { return _is_full_compaction_running; } |
474 | | void clear_cache() override; |
475 | | |
476 | 3 | int32_t get_compaction_score() const { return _compaction_score; } |
477 | | |
478 | 0 | void set_compaction_score(int32_t compaction_score) { _compaction_score = compaction_score; } |
479 | | |
480 | 10.9k | void add_compaction_score(int32_t score) { |
481 | 10.9k | if (_compaction_score < 0) { |
482 | 10.9k | return; |
483 | 10.9k | } |
484 | 21 | _compaction_score += score; |
485 | 21 | } |
486 | | |
487 | 0 | void minus_compaction_score(int32_t score) { |
488 | 0 | if (_compaction_score < 0) { |
489 | 0 | return; |
490 | 0 | } |
491 | 0 | _compaction_score -= score; |
492 | 0 | } |
493 | | |
494 | | private: |
495 | | Status _init_once_action(); |
496 | | bool _contains_rowset(const RowsetId rowset_id); |
497 | | Status _contains_version(const Version& version); |
498 | | |
499 | | // Returns: |
500 | | // version: the max continuous version from beginning |
501 | | // max_version: the max version of this tablet |
502 | | void _max_continuous_version_from_beginning_unlocked(Version* version, Version* max_version, |
503 | | bool* has_version_cross) const; |
504 | | RowsetSharedPtr _rowset_with_largest_size(); |
505 | | /// Delete stale rowset by version. This method not only delete the version in expired rowset map, |
506 | | /// but also delete the version in rowset meta vector. |
507 | | void _delete_stale_rowset_by_version(const Version& version); |
508 | | |
509 | | uint32_t _calc_cumulative_compaction_score( |
510 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
511 | | uint32_t _calc_base_compaction_score() const; |
512 | | |
513 | | std::vector<RowsetSharedPtr> _pick_visible_rowsets_to_compaction(int64_t min_start_version, |
514 | | int64_t max_start_version); |
515 | | |
516 | | void _init_context_common_fields(RowsetWriterContext& context); |
517 | | |
518 | | //////////////////////////////////////////////////////////////////////////// |
519 | | // begin cooldown functions |
520 | | //////////////////////////////////////////////////////////////////////////// |
521 | | Status _cooldown_data(RowsetSharedPtr rowset); |
522 | | Status _follow_cooldowned_data(); |
523 | | Status _read_cooldown_meta(const StorageResource& storage_resource, |
524 | | TabletMetaPB* tablet_meta_pb); |
525 | | bool _has_data_to_cooldown(); |
526 | | int64_t _get_newest_cooldown_time(const RowsetSharedPtr& rowset); |
527 | | //////////////////////////////////////////////////////////////////////////// |
528 | | // end cooldown functions |
529 | | //////////////////////////////////////////////////////////////////////////// |
530 | | |
531 | | void _clear_cache_by_rowset(const BetaRowsetSharedPtr& rowset); |
532 | | void check_table_size_correctness(); |
533 | | std::string get_segment_path(const RowsetMetaSharedPtr& rs_meta, int64_t seg_id); |
534 | | int64_t get_segment_file_size(const RowsetMetaSharedPtr& rs_meta); |
535 | | int64_t get_inverted_index_file_size(const RowsetMetaSharedPtr& rs_meta); |
536 | | |
537 | | public: |
538 | | static const int64_t K_INVALID_CUMULATIVE_POINT = -1; |
539 | | |
540 | | private: |
541 | | StorageEngine& _engine; |
542 | | DataDir* _data_dir = nullptr; |
543 | | |
544 | | std::string _tablet_path; |
545 | | |
546 | | DorisCallOnce<Status> _init_once; |
547 | | // meta store lock is used for prevent 2 threads do checkpoint concurrently |
548 | | // it will be used in econ-mode in the future |
549 | | std::shared_mutex _meta_store_lock; |
550 | | std::mutex _ingest_lock; |
551 | | std::mutex _base_compaction_lock; |
552 | | std::mutex _cumulative_compaction_lock; |
553 | | std::shared_timed_mutex _migration_lock; |
554 | | std::mutex _build_inverted_index_lock; |
555 | | |
556 | | // In unique key table with MoW, we should guarantee that only one |
557 | | // writer can update rowset and delete bitmap at the same time. |
558 | | // We use a separate lock rather than _meta_lock, to avoid blocking read queries |
559 | | // during publish_txn, which might take hundreds of milliseconds |
560 | | mutable std::mutex _rowset_update_lock; |
561 | | |
562 | | // if this tablet is broken, set to true. default is false |
563 | | std::atomic<bool> _is_bad; |
564 | | // timestamp of last cumu compaction failure |
565 | | std::atomic<int64_t> _last_cumu_compaction_failure_millis; |
566 | | // timestamp of last base compaction failure |
567 | | std::atomic<int64_t> _last_base_compaction_failure_millis; |
568 | | // timestamp of last full compaction failure |
569 | | std::atomic<int64_t> _last_full_compaction_failure_millis; |
570 | | // timestamp of last cumu compaction success |
571 | | std::atomic<int64_t> _last_cumu_compaction_success_millis; |
572 | | // timestamp of last base compaction success |
573 | | std::atomic<int64_t> _last_base_compaction_success_millis; |
574 | | // timestamp of last full compaction success |
575 | | std::atomic<int64_t> _last_full_compaction_success_millis; |
576 | | // timestamp of last base compaction schedule time |
577 | | std::atomic<int64_t> _last_base_compaction_schedule_millis; |
578 | | std::atomic<int64_t> _cumulative_point; |
579 | | std::atomic<int64_t> _cumulative_promotion_size; |
580 | | std::atomic<int32_t> _newly_created_rowset_num; |
581 | | std::atomic<int64_t> _last_checkpoint_time; |
582 | | std::string _last_base_compaction_status; |
583 | | |
584 | | // single replica compaction status |
585 | | std::string _last_single_compaction_failure_status; |
586 | | Version _last_fetched_version; |
587 | | |
588 | | // cumulative compaction policy |
589 | | std::shared_ptr<CumulativeCompactionPolicy> _cumulative_compaction_policy; |
590 | | std::string_view _cumulative_compaction_type; |
591 | | |
592 | | // use a separate thread to check all tablets paths existence |
593 | | std::atomic<bool> _is_tablet_path_exists; |
594 | | |
595 | | int64_t _last_missed_version; |
596 | | int64_t _last_missed_time_s; |
597 | | |
598 | | bool _skip_cumu_compaction = false; |
599 | | int64_t _skip_cumu_compaction_ts; |
600 | | |
601 | | bool _skip_base_compaction = false; |
602 | | int64_t _skip_base_compaction_ts; |
603 | | |
604 | | // cooldown related |
605 | | CooldownConf _cooldown_conf; |
606 | | // `_cooldown_conf_lock` is used to serialize update cooldown conf and all operations that: |
607 | | // 1. read cooldown conf |
608 | | // 2. upload rowsets to remote storage |
609 | | // 3. update cooldown meta id |
610 | | mutable std::shared_mutex _cooldown_conf_lock; |
611 | | // `_cold_compaction_lock` is used to serialize cold data compaction and all operations that |
612 | | // may delete compaction input rowsets. |
613 | | std::mutex _cold_compaction_lock; |
614 | | int64_t _last_failed_follow_cooldown_time = 0; |
615 | | // `_alter_failed` is used to indicate whether the tablet failed to perform a schema change |
616 | | std::atomic<bool> _alter_failed = false; |
617 | | |
618 | | int64_t _io_error_times = 0; |
619 | | |
620 | | // partition's visible version. it sync from fe, but not real-time. |
621 | | std::shared_ptr<const VersionWithTime> _visible_version; |
622 | | |
623 | | std::atomic_bool _is_full_compaction_running = false; |
624 | | |
625 | | int32_t _compaction_score = -1; |
626 | | int32_t _score_check_cnt = 0; |
627 | | }; |
628 | | |
629 | 10 | inline CumulativeCompactionPolicy* Tablet::cumulative_compaction_policy() { |
630 | 10 | return _cumulative_compaction_policy.get(); |
631 | 10 | } |
632 | | |
633 | 271 | inline bool Tablet::init_succeeded() { |
634 | 271 | return _init_once.has_called() && _init_once.stored_result().ok(); |
635 | 271 | } |
636 | | |
637 | 259 | inline bool Tablet::is_used() { |
638 | 259 | return !_is_bad && _data_dir->is_used(); |
639 | 259 | } |
640 | | |
641 | 303 | inline void Tablet::register_tablet_into_dir() { |
642 | 303 | _data_dir->register_tablet(this); |
643 | 303 | } |
644 | | |
645 | 255 | inline void Tablet::deregister_tablet_from_dir() { |
646 | 255 | _data_dir->deregister_tablet(this); |
647 | 255 | } |
648 | | |
649 | 37 | inline int64_t Tablet::cumulative_layer_point() const { |
650 | 37 | return _cumulative_point; |
651 | 37 | } |
652 | | |
653 | 328 | inline void Tablet::set_cumulative_layer_point(int64_t new_point) { |
654 | | // cumulative point should only be reset to -1, or be increased |
655 | 328 | CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) |
656 | 0 | << "Unexpected cumulative point: " << new_point |
657 | 0 | << ", origin: " << _cumulative_point.load(); |
658 | 328 | _cumulative_point = new_point; |
659 | 328 | } |
660 | | |
661 | 20 | inline int64_t Tablet::cumulative_promotion_size() const { |
662 | 20 | return _cumulative_promotion_size; |
663 | 20 | } |
664 | | |
665 | 47 | inline void Tablet::set_cumulative_promotion_size(int64_t new_size) { |
666 | 47 | _cumulative_promotion_size = new_size; |
667 | 47 | } |
668 | | |
669 | | // TODO(lingbin): Why other methods that need to get information from _tablet_meta |
670 | | // are not locked, here needs a comment to explain. |
671 | 8 | inline size_t Tablet::tablet_footprint() { |
672 | 8 | std::shared_lock rdlock(_meta_lock); |
673 | 8 | return _tablet_meta->tablet_footprint(); |
674 | 8 | } |
675 | | |
676 | 0 | inline size_t Tablet::tablet_local_size() { |
677 | 0 | std::shared_lock rdlock(_meta_lock); |
678 | 0 | return _tablet_meta->tablet_local_size(); |
679 | 0 | } |
680 | | |
681 | 0 | inline size_t Tablet::tablet_remote_size() { |
682 | 0 | std::shared_lock rdlock(_meta_lock); |
683 | 0 | return _tablet_meta->tablet_remote_size(); |
684 | 0 | } |
685 | | |
686 | | // TODO(lingbin): Why other methods which need to get information from _tablet_meta |
687 | | // are not locked, here needs a comment to explain. |
688 | 18 | inline size_t Tablet::num_rows() { |
689 | 18 | std::shared_lock rdlock(_meta_lock); |
690 | 18 | return _tablet_meta->num_rows(); |
691 | 18 | } |
692 | | |
693 | 27 | inline int Tablet::version_count() const { |
694 | 27 | std::shared_lock rdlock(_meta_lock); |
695 | 27 | return _tablet_meta->version_count(); |
696 | 27 | } |
697 | | |
698 | 27 | inline int Tablet::stale_version_count() const { |
699 | 27 | std::shared_lock rdlock(_meta_lock); |
700 | 27 | return _tablet_meta->stale_version_count(); |
701 | 27 | } |
702 | | |
703 | 34 | inline Version Tablet::max_version() const { |
704 | 34 | std::shared_lock rdlock(_meta_lock); |
705 | 34 | return _tablet_meta->max_version(); |
706 | 34 | } |
707 | | |
708 | 0 | inline uint64_t Tablet::segment_count() const { |
709 | 0 | std::shared_lock rdlock(_meta_lock); |
710 | 0 | uint64_t segment_nums = 0; |
711 | 0 | for (const auto& rs_meta : _tablet_meta->all_rs_metas()) { |
712 | 0 | segment_nums += rs_meta->num_segments(); |
713 | 0 | } |
714 | 0 | return segment_nums; |
715 | 0 | } |
716 | | |
717 | 0 | inline SortType Tablet::sort_type() const { |
718 | 0 | return _tablet_meta->tablet_schema()->sort_type(); |
719 | 0 | } |
720 | | |
721 | 0 | inline size_t Tablet::sort_col_num() const { |
722 | 0 | return _tablet_meta->tablet_schema()->sort_col_num(); |
723 | 0 | } |
724 | | |
725 | 0 | inline size_t Tablet::num_columns() const { |
726 | 0 | return _tablet_meta->tablet_schema()->num_columns(); |
727 | 0 | } |
728 | | |
729 | 0 | inline size_t Tablet::num_null_columns() const { |
730 | 0 | return _tablet_meta->tablet_schema()->num_null_columns(); |
731 | 0 | } |
732 | | |
733 | 0 | inline size_t Tablet::num_short_key_columns() const { |
734 | 0 | return _tablet_meta->tablet_schema()->num_short_key_columns(); |
735 | 0 | } |
736 | | |
737 | 0 | inline size_t Tablet::num_rows_per_row_block() const { |
738 | 0 | return _tablet_meta->tablet_schema()->num_rows_per_row_block(); |
739 | 0 | } |
740 | | |
741 | 0 | inline CompressKind Tablet::compress_kind() const { |
742 | 0 | return _tablet_meta->tablet_schema()->compress_kind(); |
743 | 0 | } |
744 | | |
745 | 0 | inline double Tablet::bloom_filter_fpp() const { |
746 | 0 | return _tablet_meta->tablet_schema()->bloom_filter_fpp(); |
747 | 0 | } |
748 | | |
749 | 0 | inline size_t Tablet::next_unique_id() const { |
750 | 0 | return _tablet_meta->tablet_schema()->next_column_unique_id(); |
751 | 0 | } |
752 | | |
753 | 0 | inline size_t Tablet::row_size() const { |
754 | 0 | return _tablet_meta->tablet_schema()->row_size(); |
755 | 0 | } |
756 | | |
757 | 27 | inline int64_t Tablet::avg_rs_meta_serialize_size() const { |
758 | 27 | return _tablet_meta->avg_rs_meta_serialize_size(); |
759 | 27 | } |
760 | | |
761 | | } // namespace doris |