/root/doris/be/src/olap/tablet.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <butil/macros.h> |
21 | | #include <glog/logging.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <cstddef> |
25 | | #include <cstdint> |
26 | | #include <functional> |
27 | | #include <memory> |
28 | | #include <mutex> |
29 | | #include <ostream> |
30 | | #include <set> |
31 | | #include <shared_mutex> |
32 | | #include <string> |
33 | | #include <string_view> |
34 | | #include <utility> |
35 | | #include <vector> |
36 | | |
37 | | #include "common/config.h" |
38 | | #include "common/status.h" |
39 | | #include "olap/base_tablet.h" |
40 | | #include "olap/binlog_config.h" |
41 | | #include "olap/data_dir.h" |
42 | | #include "olap/olap_common.h" |
43 | | #include "olap/partial_update_info.h" |
44 | | #include "olap/rowset/rowset.h" |
45 | | #include "olap/rowset/rowset_meta.h" |
46 | | #include "olap/rowset/rowset_reader.h" |
47 | | #include "olap/rowset/segment_v2/segment.h" |
48 | | #include "olap/version_graph.h" |
49 | | #include "segment_loader.h" |
50 | | #include "util/metrics.h" |
51 | | #include "util/once.h" |
52 | | #include "util/slice.h" |
53 | | |
54 | | namespace bvar { |
55 | | template <typename T> |
56 | | class Adder; |
57 | | } |
58 | | |
59 | | namespace doris { |
60 | | |
61 | | class Tablet; |
62 | | class CumulativeCompactionPolicy; |
63 | | class CompactionMixin; |
64 | | class SingleReplicaCompaction; |
65 | | class RowsetWriter; |
66 | | struct RowsetWriterContext; |
67 | | class TTabletInfo; |
68 | | class TabletMetaPB; |
69 | | class TupleDescriptor; |
70 | | class CalcDeleteBitmapToken; |
71 | | enum CompressKind : int; |
72 | | class RowsetBinlogMetasPB; |
73 | | |
74 | | namespace io { |
75 | | class RemoteFileSystem; |
76 | | } // namespace io |
77 | | namespace vectorized { |
78 | | class Block; |
79 | | } // namespace vectorized |
80 | | struct RowLocation; |
81 | | enum KeysType : int; |
82 | | enum SortType : int; |
83 | | |
84 | | enum TabletStorageType { STORAGE_TYPE_LOCAL, STORAGE_TYPE_REMOTE, STORAGE_TYPE_REMOTE_AND_LOCAL }; |
85 | | |
86 | | extern bvar::Adder<uint64_t> unused_remote_rowset_num; |
87 | | |
88 | | static inline constexpr auto TRACE_TABLET_LOCK_THRESHOLD = std::chrono::seconds(1); |
89 | | |
90 | | struct WriteCooldownMetaExecutors { |
91 | | WriteCooldownMetaExecutors(size_t executor_nums = 5); |
92 | | |
93 | | void stop(); |
94 | | |
95 | | void submit(TabletSharedPtr tablet); |
96 | 5 | size_t _get_executor_pos(int64_t tablet_id) const { |
97 | 5 | return std::hash<int64_t>()(tablet_id) % _executor_nums; |
98 | 5 | }; |
99 | | // Each executor is a mpsc to ensure uploads of the same tablet meta are not concurrent |
100 | | // FIXME(AlexYue): Use mpsc instead of `ThreadPool` with 1 thread |
101 | | // We use PriorityThreadPool since it would call status inside it's `shutdown` function. |
102 | | // Consider one situation where the StackTraceCache's singleton is detructed before |
103 | | // this WriteCooldownMetaExecutors's singleton, then invoking the status would also call |
104 | | // StackTraceCache which would then result in heap use after free like #23834 |
105 | | std::vector<std::unique_ptr<PriorityThreadPool>> _executors; |
106 | | std::unordered_set<int64_t> _pending_tablets; |
107 | | std::mutex _latch; |
108 | | size_t _executor_nums; |
109 | | }; |
110 | | |
111 | | class Tablet final : public BaseTablet { |
112 | | public: |
113 | | Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* data_dir, |
114 | | const std::string_view& cumulative_compaction_type = ""); |
115 | | |
116 | 2.48k | DataDir* data_dir() const { return _data_dir; } |
117 | 58 | int64_t replica_id() const { return _tablet_meta->replica_id(); } |
118 | | |
119 | 724 | const std::string& tablet_path() const { return _tablet_path; } |
120 | | |
121 | | bool set_tablet_schema_into_rowset_meta(); |
122 | | Status init(); |
123 | | bool init_succeeded(); |
124 | | |
125 | | bool is_used(); |
126 | | |
127 | | void register_tablet_into_dir(); |
128 | | void deregister_tablet_from_dir(); |
129 | | |
130 | | void save_meta(); |
131 | | // Used in clone task, to update local meta when finishing a clone job |
132 | | Status revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add, |
133 | | const std::vector<RowsetSharedPtr>& to_delete, |
134 | | bool is_incremental_clone); |
135 | | |
136 | | int64_t cumulative_layer_point() const; |
137 | | void set_cumulative_layer_point(int64_t new_point); |
138 | | inline int64_t cumulative_promotion_size() const; |
139 | | inline void set_cumulative_promotion_size(int64_t new_size); |
140 | | |
141 | | // Disk space occupied by tablet, contain local and remote. |
142 | | size_t tablet_footprint() override; |
143 | | // Local disk space occupied by tablet. |
144 | | size_t tablet_local_size(); |
145 | | // Remote disk space occupied by tablet. |
146 | | size_t tablet_remote_size(); |
147 | | |
148 | | size_t num_rows(); |
149 | | int version_count() const; |
150 | | int stale_version_count() const; |
151 | | bool exceed_version_limit(int32_t limit) override; |
152 | | uint64_t segment_count() const; |
153 | | Version max_version() const; |
154 | | CumulativeCompactionPolicy* cumulative_compaction_policy(); |
155 | | |
156 | | // properties encapsulated in TabletSchema |
157 | | SortType sort_type() const; |
158 | | size_t sort_col_num() const; |
159 | | size_t num_columns() const; |
160 | | size_t num_null_columns() const; |
161 | | size_t num_short_key_columns() const; |
162 | | size_t num_rows_per_row_block() const; |
163 | | CompressKind compress_kind() const; |
164 | | double bloom_filter_fpp() const; |
165 | | size_t next_unique_id() const; |
166 | | size_t row_size() const; |
167 | | int64_t avg_rs_meta_serialize_size() const; |
168 | | |
169 | | // operation in rowsets |
170 | | Status add_rowset(RowsetSharedPtr rowset); |
171 | | Status create_initial_rowset(const int64_t version); |
172 | | |
173 | | // MUST hold EXCLUSIVE `_meta_lock`. |
174 | | Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add, |
175 | | std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false); |
176 | | |
177 | | Status add_inc_rowset(const RowsetSharedPtr& rowset); |
178 | | /// Delete stale rowset by timing. This delete policy uses now() minutes |
179 | | /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset |
180 | | /// to delete. When rowset is deleted, it will be added to StorageEngine unused map and record |
181 | | /// need to delete flag. |
182 | | void delete_expired_stale_rowset(); |
183 | | |
184 | | // Given spec_version, find a continuous version path and store it in version_path. |
185 | | // If quiet is true, then only "does this path exist" is returned. |
186 | | // If skip_missing_version is true, return ok even there are missing versions. |
187 | | Status capture_consistent_versions_unlocked(const Version& spec_version, Versions* version_path, |
188 | | bool skip_missing_version, bool quiet) const; |
189 | | |
190 | | // if quiet is true, no error log will be printed if there are missing versions |
191 | | Status check_version_integrity(const Version& version, bool quiet = false); |
192 | | bool check_version_exist(const Version& version) const; |
193 | | void acquire_version_and_rowsets( |
194 | | std::vector<std::pair<Version, RowsetSharedPtr>>* version_rowsets) const; |
195 | | |
196 | | Status capture_consistent_rowsets_unlocked( |
197 | | const Version& spec_version, std::vector<RowsetSharedPtr>* rowsets) const override; |
198 | | |
199 | | // If skip_missing_version is true, skip versions if they are missing. |
200 | | Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits, |
201 | | bool skip_missing_version) override; |
202 | | |
203 | | // Find the missed versions until the spec_version. |
204 | | // |
205 | | // for example: |
206 | | // [0-4][5-5][8-8][9-9][14-14] |
207 | | // if spec_version = 12, it will return [6, 6], [7, 7], [10, 10], [11, 11], [12, 12] |
208 | | Versions calc_missed_versions(int64_t spec_version, Versions existing_versions) const override; |
209 | | |
210 | | // meta lock |
211 | 129 | std::shared_mutex& get_header_lock() { return _meta_lock; } |
212 | 1 | std::mutex& get_rowset_update_lock() { return _rowset_update_lock; } |
213 | 29 | std::mutex& get_push_lock() { return _ingest_lock; } |
214 | 2 | std::mutex& get_base_compaction_lock() { return _base_compaction_lock; } |
215 | 63 | std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; } |
216 | | |
217 | 30 | std::shared_timed_mutex& get_migration_lock() { return _migration_lock; } |
218 | | |
219 | 0 | std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; } |
220 | | |
221 | | // operation for compaction |
222 | | bool can_do_compaction(size_t path_hash, CompactionType compaction_type); |
223 | | bool suitable_for_compaction( |
224 | | CompactionType compaction_type, |
225 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
226 | | |
227 | | uint32_t calc_compaction_score(); |
228 | | |
229 | | // This function to find max continuous version from the beginning. |
230 | | // For example: If there are 1, 2, 3, 5, 6, 7 versions belongs tablet, then 3 is target. |
231 | | // 3 will be saved in "version", and 7 will be saved in "max_version", if max_version != nullptr |
232 | | void max_continuous_version_from_beginning(Version* version, Version* max_version = nullptr); |
233 | | |
234 | 0 | void set_bad(bool is_bad) { _is_bad = is_bad; } |
235 | | |
236 | 51 | int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; } |
237 | 0 | void set_last_cumu_compaction_failure_time(int64_t millis) { |
238 | 0 | _last_cumu_compaction_failure_millis = millis; |
239 | 0 | } |
240 | | |
241 | 0 | int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; } |
242 | 0 | void set_last_base_compaction_failure_time(int64_t millis) { |
243 | 0 | _last_base_compaction_failure_millis = millis; |
244 | 0 | } |
245 | | |
246 | 0 | int64_t last_full_compaction_failure_time() { return _last_full_compaction_failure_millis; } |
247 | 0 | void set_last_full_compaction_failure_time(int64_t millis) { |
248 | 0 | _last_full_compaction_failure_millis = millis; |
249 | 0 | } |
250 | | |
251 | 4 | int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; } |
252 | 2 | void set_last_cumu_compaction_success_time(int64_t millis) { |
253 | 2 | _last_cumu_compaction_success_millis = millis; |
254 | 2 | } |
255 | | |
256 | 0 | int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; } |
257 | 0 | void set_last_base_compaction_success_time(int64_t millis) { |
258 | 0 | _last_base_compaction_success_millis = millis; |
259 | 0 | } |
260 | | |
261 | 0 | int64_t last_full_compaction_success_time() { return _last_full_compaction_success_millis; } |
262 | 0 | void set_last_full_compaction_success_time(int64_t millis) { |
263 | 0 | _last_full_compaction_success_millis = millis; |
264 | 0 | } |
265 | | |
266 | 0 | int64_t last_base_compaction_schedule_time() { return _last_base_compaction_schedule_millis; } |
267 | 0 | void set_last_base_compaction_schedule_time(int64_t millis) { |
268 | 0 | _last_base_compaction_schedule_millis = millis; |
269 | 0 | } |
270 | | |
271 | 0 | void set_last_single_compaction_failure_status(std::string status) { |
272 | 0 | _last_single_compaction_failure_status = std::move(status); |
273 | 0 | } |
274 | | |
275 | 0 | void set_last_fetched_version(Version version) { _last_fetched_version = std::move(version); } |
276 | | |
277 | | void delete_all_files(); |
278 | | |
279 | | void check_tablet_path_exists(); |
280 | | |
281 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_cumulative_compaction(); |
282 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction(); |
283 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction(); |
284 | | std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_build_inverted_index( |
285 | | const std::set<int64_t>& alter_index_uids, bool is_drop_op); |
286 | | |
287 | | // used for single compaction to get the local versions |
288 | | // Single compaction does not require remote rowsets and cannot violate the cooldown semantics |
289 | | std::vector<Version> get_all_local_versions(); |
290 | | |
291 | | void calculate_cumulative_point(); |
292 | | // TODO(ygl): |
293 | 0 | bool is_primary_replica() { return false; } |
294 | | |
295 | | // return true if the checkpoint is actually done |
296 | | bool do_tablet_meta_checkpoint(); |
297 | | |
298 | | // Check whether the rowset is useful or not, unuseful rowset can be swept up then. |
299 | | // Rowset which is under tablet's management is useful, i.e. rowset is in |
300 | | // _rs_version_map, or _stale_rs_version_map. |
301 | | // Rowset whose version range is not covered by this tablet is also useful. |
302 | | bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta); |
303 | | |
304 | | void build_tablet_report_info(TTabletInfo* tablet_info, |
305 | | bool enable_consecutive_missing_check = false, |
306 | | bool enable_path_check = false); |
307 | | |
308 | | // return a json string to show the compaction status of this tablet |
309 | | void get_compaction_status(std::string* json_result); |
310 | | |
311 | | static Status prepare_compaction_and_calculate_permits( |
312 | | CompactionType compaction_type, const TabletSharedPtr& tablet, |
313 | | std::shared_ptr<CompactionMixin>& compaction, int64_t& permits); |
314 | | |
315 | | void execute_compaction(CompactionMixin& compaction); |
316 | | void execute_single_replica_compaction(SingleReplicaCompaction& compaction); |
317 | | |
318 | | void set_cumulative_compaction_policy( |
319 | 0 | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) { |
320 | 0 | _cumulative_compaction_policy = cumulative_compaction_policy; |
321 | 0 | } |
322 | | |
323 | 0 | std::shared_ptr<CumulativeCompactionPolicy> get_cumulative_compaction_policy() { |
324 | 0 | return _cumulative_compaction_policy; |
325 | 0 | } |
326 | | |
327 | 0 | void set_last_base_compaction_status(std::string status) { |
328 | 0 | _last_base_compaction_status = std::move(status); |
329 | 0 | } |
330 | | |
331 | 0 | std::string get_last_base_compaction_status() { return _last_base_compaction_status; } |
332 | | |
333 | | std::tuple<int64_t, int64_t> get_visible_version_and_time() const; |
334 | | |
335 | 139 | void set_visible_version(const std::shared_ptr<const VersionWithTime>& visible_version) { |
336 | 139 | std::atomic_store_explicit(&_visible_version, visible_version, std::memory_order_relaxed); |
337 | 139 | } |
338 | | |
339 | | bool should_fetch_from_peer(); |
340 | | |
341 | 0 | inline bool all_beta() const { |
342 | 0 | std::shared_lock rdlock(_meta_lock); |
343 | 0 | return _tablet_meta->all_beta(); |
344 | 0 | } |
345 | | |
346 | 0 | const TabletSchemaSPtr& tablet_schema_unlocked() const { return _max_version_schema; } |
347 | | |
348 | | Result<std::unique_ptr<RowsetWriter>> create_rowset_writer(RowsetWriterContext& context, |
349 | | bool vertical) override; |
350 | | |
351 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
352 | | const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info, |
353 | | int64_t txn_expiration = 0) override; |
354 | | Result<std::unique_ptr<RowsetWriter>> create_transient_rowset_writer( |
355 | | RowsetWriterContext& context, const RowsetId& rowset_id); |
356 | | |
357 | | Status create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset); |
358 | | |
359 | | // MUST hold EXCLUSIVE `_meta_lock` |
360 | | void add_rowsets(const std::vector<RowsetSharedPtr>& to_add); |
361 | | // MUST hold EXCLUSIVE `_meta_lock` |
362 | | Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool move_to_stale); |
363 | | |
364 | | // MUST hold SHARED `_meta_lock` |
365 | 70 | const auto& rowset_map() const { return _rs_version_map; } |
366 | | // MUST hold SHARED `_meta_lock` |
367 | 69 | const auto& stale_rowset_map() const { return _stale_rs_version_map; } |
368 | | |
369 | | //////////////////////////////////////////////////////////////////////////// |
370 | | // begin cooldown functions |
371 | | //////////////////////////////////////////////////////////////////////////// |
372 | 22 | int64_t storage_policy_id() const { return _tablet_meta->storage_policy_id(); } |
373 | 4 | void set_storage_policy_id(int64_t id) { _tablet_meta->set_storage_policy_id(id); } |
374 | | |
375 | 0 | int64_t last_failed_follow_cooldown_time() const { return _last_failed_follow_cooldown_time; } |
376 | | |
377 | | // Cooldown to remote fs. |
378 | | Status cooldown(RowsetSharedPtr rowset = nullptr); |
379 | | |
380 | | RowsetSharedPtr pick_cooldown_rowset(); |
381 | | |
382 | | RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size); |
383 | | |
384 | | struct CooldownConf { |
385 | | int64_t term = -1; |
386 | | int64_t cooldown_replica_id = -1; |
387 | | }; |
388 | | |
389 | 0 | CooldownConf cooldown_conf() const { |
390 | 0 | std::shared_lock rlock(_cooldown_conf_lock); |
391 | 0 | return _cooldown_conf; |
392 | 0 | } |
393 | | |
394 | 0 | CooldownConf cooldown_conf_unlocked() const { return _cooldown_conf; } |
395 | | |
396 | | // Return `true` if update success |
397 | | bool update_cooldown_conf(int64_t cooldown_term, int64_t cooldown_replica_id); |
398 | | |
399 | | Status remove_all_remote_rowsets(); |
400 | | |
401 | | void record_unused_remote_rowset(const RowsetId& rowset_id, const std::string& resource, |
402 | | int64_t num_segments); |
403 | | |
404 | | uint32_t calc_cold_data_compaction_score() const; |
405 | | |
406 | 0 | std::mutex& get_cold_compaction_lock() { return _cold_compaction_lock; } |
407 | | |
408 | 0 | std::shared_mutex& get_cooldown_conf_lock() { return _cooldown_conf_lock; } |
409 | | |
410 | | static void async_write_cooldown_meta(TabletSharedPtr tablet); |
411 | | // Return `ABORTED` if should not to retry again |
412 | | Status write_cooldown_meta(); |
413 | | //////////////////////////////////////////////////////////////////////////// |
414 | | // end cooldown functions |
415 | | //////////////////////////////////////////////////////////////////////////// |
416 | | |
417 | | CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override; |
418 | | Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, |
419 | | DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer, |
420 | | const RowsetIdUnorderedSet& cur_rowset_ids) override; |
421 | | |
422 | | void merge_delete_bitmap(const DeleteBitmap& delete_bitmap); |
423 | | bool check_all_rowset_segment(); |
424 | | |
425 | | void update_max_version_schema(const TabletSchemaSPtr& tablet_schema); |
426 | | |
427 | | void set_skip_compaction(bool skip, |
428 | | CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION, |
429 | | int64_t start = -1); |
430 | | bool should_skip_compaction(CompactionType compaction_type, int64_t now); |
431 | | |
432 | | std::vector<std::string> get_binlog_filepath(std::string_view binlog_version) const; |
433 | | std::pair<std::string, int64_t> get_binlog_info(std::string_view binlog_version) const; |
434 | | std::string get_rowset_binlog_meta(std::string_view binlog_version, |
435 | | std::string_view rowset_id) const; |
436 | | Status get_rowset_binlog_metas(const std::vector<int64_t>& binlog_versions, |
437 | | RowsetBinlogMetasPB* metas_pb); |
438 | | Status get_rowset_binlog_metas(Version binlog_versions, RowsetBinlogMetasPB* metas_pb); |
439 | | std::string get_segment_filepath(std::string_view rowset_id, |
440 | | std::string_view segment_index) const; |
441 | | std::string get_segment_filepath(std::string_view rowset_id, int64_t segment_index) const; |
442 | | std::string get_segment_index_filepath(std::string_view rowset_id, |
443 | | std::string_view segment_index, |
444 | | std::string_view index_id) const; |
445 | | std::string get_segment_index_filepath(std::string_view rowset_id, int64_t segment_index, |
446 | | int64_t index_id) const; |
447 | | bool can_add_binlog(uint64_t total_binlog_size) const; |
448 | | void gc_binlogs(int64_t version); |
449 | | Status ingest_binlog_metas(RowsetBinlogMetasPB* metas_pb); |
450 | | |
451 | | void report_error(const Status& st); |
452 | | |
453 | 0 | inline int64_t get_io_error_times() const { return _io_error_times; } |
454 | | |
455 | 0 | inline bool is_io_error_too_times() const { |
456 | 0 | return config::max_tablet_io_errors > 0 && _io_error_times >= config::max_tablet_io_errors; |
457 | 0 | } |
458 | | |
459 | 0 | int64_t get_table_id() { return _tablet_meta->table_id(); } |
460 | | |
461 | | // binlog related functions |
462 | | bool is_enable_binlog(); |
463 | 0 | bool is_binlog_enabled() { return _tablet_meta->binlog_config().is_enable(); } |
464 | 0 | int64_t binlog_ttl_ms() const { return _tablet_meta->binlog_config().ttl_seconds(); } |
465 | 0 | int64_t binlog_max_bytes() const { return _tablet_meta->binlog_config().max_bytes(); } |
466 | | |
467 | | void set_binlog_config(BinlogConfig binlog_config); |
468 | | |
469 | 0 | void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; } |
470 | 0 | bool is_alter_failed() { return _alter_failed; } |
471 | | |
472 | 0 | void set_is_full_compaction_running(bool is_full_compaction_running) { |
473 | 0 | _is_full_compaction_running = is_full_compaction_running; |
474 | 0 | } |
475 | 0 | inline bool is_full_compaction_running() const { return _is_full_compaction_running; } |
476 | | void clear_cache() override; |
477 | | |
478 | 3 | int32_t get_compaction_score() const { return _compaction_score; } |
479 | | |
480 | 0 | void set_compaction_score(int32_t compaction_score) { _compaction_score = compaction_score; } |
481 | | |
482 | 506 | void add_compaction_score(int32_t score) { |
483 | 506 | if (_compaction_score < 0) { |
484 | 485 | return; |
485 | 485 | } |
486 | 21 | _compaction_score += score; |
487 | 21 | } |
488 | | |
489 | 0 | void minus_compaction_score(int32_t score) { |
490 | 0 | if (_compaction_score < 0) { |
491 | 0 | return; |
492 | 0 | } |
493 | 0 | _compaction_score -= score; |
494 | 0 | } |
495 | | |
496 | | private: |
497 | | Status _init_once_action(); |
498 | | bool _contains_rowset(const RowsetId rowset_id); |
499 | | Status _contains_version(const Version& version); |
500 | | |
501 | | // Returns: |
502 | | // version: the max continuous version from beginning |
503 | | // max_version: the max version of this tablet |
504 | | void _max_continuous_version_from_beginning_unlocked(Version* version, Version* max_version, |
505 | | bool* has_version_cross) const; |
506 | | RowsetSharedPtr _rowset_with_largest_size(); |
507 | | /// Delete stale rowset by version. This method not only delete the version in expired rowset map, |
508 | | /// but also delete the version in rowset meta vector. |
509 | | void _delete_stale_rowset_by_version(const Version& version); |
510 | | |
511 | | uint32_t _calc_cumulative_compaction_score( |
512 | | std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy); |
513 | | uint32_t _calc_base_compaction_score() const; |
514 | | |
515 | | std::vector<RowsetSharedPtr> _pick_visible_rowsets_to_compaction(int64_t min_start_version, |
516 | | int64_t max_start_version); |
517 | | |
518 | | void _init_context_common_fields(RowsetWriterContext& context); |
519 | | |
520 | | //////////////////////////////////////////////////////////////////////////// |
521 | | // begin cooldown functions |
522 | | //////////////////////////////////////////////////////////////////////////// |
523 | | Status _cooldown_data(RowsetSharedPtr rowset); |
524 | | Status _follow_cooldowned_data(); |
525 | | Status _read_cooldown_meta(const StorageResource& storage_resource, |
526 | | TabletMetaPB* tablet_meta_pb); |
527 | | bool _has_data_to_cooldown(); |
528 | | int64_t _get_newest_cooldown_time(const RowsetSharedPtr& rowset); |
529 | | //////////////////////////////////////////////////////////////////////////// |
530 | | // end cooldown functions |
531 | | //////////////////////////////////////////////////////////////////////////// |
532 | | |
533 | | void _clear_cache_by_rowset(const BetaRowsetSharedPtr& rowset); |
534 | | void check_table_size_correctness(); |
535 | | std::string get_segment_path(const RowsetMetaSharedPtr& rs_meta, int64_t seg_id); |
536 | | int64_t get_segment_file_size(const RowsetMetaSharedPtr& rs_meta); |
537 | | int64_t get_inverted_index_file_szie(const RowsetMetaSharedPtr& rs_meta); |
538 | | |
539 | | public: |
540 | | static const int64_t K_INVALID_CUMULATIVE_POINT = -1; |
541 | | |
542 | | private: |
543 | | StorageEngine& _engine; |
544 | | DataDir* _data_dir = nullptr; |
545 | | |
546 | | std::string _tablet_path; |
547 | | |
548 | | DorisCallOnce<Status> _init_once; |
549 | | // meta store lock is used for prevent 2 threads do checkpoint concurrently |
550 | | // it will be used in econ-mode in the future |
551 | | std::shared_mutex _meta_store_lock; |
552 | | std::mutex _ingest_lock; |
553 | | std::mutex _base_compaction_lock; |
554 | | std::mutex _cumulative_compaction_lock; |
555 | | std::shared_timed_mutex _migration_lock; |
556 | | std::mutex _build_inverted_index_lock; |
557 | | |
558 | | // In unique key table with MoW, we should guarantee that only one |
559 | | // writer can update rowset and delete bitmap at the same time. |
560 | | // We use a separate lock rather than _meta_lock, to avoid blocking read queries |
561 | | // during publish_txn, which might take hundreds of milliseconds |
562 | | mutable std::mutex _rowset_update_lock; |
563 | | |
564 | | // if this tablet is broken, set to true. default is false |
565 | | std::atomic<bool> _is_bad; |
566 | | // timestamp of last cumu compaction failure |
567 | | std::atomic<int64_t> _last_cumu_compaction_failure_millis; |
568 | | // timestamp of last base compaction failure |
569 | | std::atomic<int64_t> _last_base_compaction_failure_millis; |
570 | | // timestamp of last full compaction failure |
571 | | std::atomic<int64_t> _last_full_compaction_failure_millis; |
572 | | // timestamp of last cumu compaction success |
573 | | std::atomic<int64_t> _last_cumu_compaction_success_millis; |
574 | | // timestamp of last base compaction success |
575 | | std::atomic<int64_t> _last_base_compaction_success_millis; |
576 | | // timestamp of last full compaction success |
577 | | std::atomic<int64_t> _last_full_compaction_success_millis; |
578 | | // timestamp of last base compaction schedule time |
579 | | std::atomic<int64_t> _last_base_compaction_schedule_millis; |
580 | | std::atomic<int64_t> _cumulative_point; |
581 | | std::atomic<int64_t> _cumulative_promotion_size; |
582 | | std::atomic<int32_t> _newly_created_rowset_num; |
583 | | std::atomic<int64_t> _last_checkpoint_time; |
584 | | std::string _last_base_compaction_status; |
585 | | |
586 | | // single replica compaction status |
587 | | std::string _last_single_compaction_failure_status; |
588 | | Version _last_fetched_version; |
589 | | |
590 | | // cumulative compaction policy |
591 | | std::shared_ptr<CumulativeCompactionPolicy> _cumulative_compaction_policy; |
592 | | std::string_view _cumulative_compaction_type; |
593 | | |
594 | | // use a seperate thread to check all tablets paths existance |
595 | | std::atomic<bool> _is_tablet_path_exists; |
596 | | |
597 | | int64_t _last_missed_version; |
598 | | int64_t _last_missed_time_s; |
599 | | |
600 | | bool _skip_cumu_compaction = false; |
601 | | int64_t _skip_cumu_compaction_ts; |
602 | | |
603 | | bool _skip_base_compaction = false; |
604 | | int64_t _skip_base_compaction_ts; |
605 | | |
606 | | // cooldown related |
607 | | CooldownConf _cooldown_conf; |
608 | | // `_cooldown_conf_lock` is used to serialize update cooldown conf and all operations that: |
609 | | // 1. read cooldown conf |
610 | | // 2. upload rowsets to remote storage |
611 | | // 3. update cooldown meta id |
612 | | mutable std::shared_mutex _cooldown_conf_lock; |
613 | | // `_cold_compaction_lock` is used to serialize cold data compaction and all operations that |
614 | | // may delete compaction input rowsets. |
615 | | std::mutex _cold_compaction_lock; |
616 | | int64_t _last_failed_follow_cooldown_time = 0; |
617 | | // `_alter_failed` is used to indicate whether the tablet failed to perform a schema change |
618 | | std::atomic<bool> _alter_failed = false; |
619 | | |
620 | | int64_t _io_error_times = 0; |
621 | | |
622 | | // partition's visible version. it sync from fe, but not real-time. |
623 | | std::shared_ptr<const VersionWithTime> _visible_version; |
624 | | |
625 | | std::atomic_bool _is_full_compaction_running = false; |
626 | | |
627 | | int32_t _compaction_score = -1; |
628 | | int32_t _score_check_cnt = 0; |
629 | | }; |
630 | | |
631 | 10 | inline CumulativeCompactionPolicy* Tablet::cumulative_compaction_policy() { |
632 | 10 | return _cumulative_compaction_policy.get(); |
633 | 10 | } |
634 | | |
635 | 65 | inline bool Tablet::init_succeeded() { |
636 | 65 | return _init_once.has_called() && _init_once.stored_result().ok(); |
637 | 65 | } |
638 | | |
639 | 53 | inline bool Tablet::is_used() { |
640 | 53 | return !_is_bad && _data_dir->is_used(); |
641 | 53 | } |
642 | | |
643 | 93 | inline void Tablet::register_tablet_into_dir() { |
644 | 93 | _data_dir->register_tablet(this); |
645 | 93 | } |
646 | | |
647 | 46 | inline void Tablet::deregister_tablet_from_dir() { |
648 | 46 | _data_dir->deregister_tablet(this); |
649 | 46 | } |
650 | | |
651 | 11 | inline int64_t Tablet::cumulative_layer_point() const { |
652 | 11 | return _cumulative_point; |
653 | 11 | } |
654 | | |
655 | 120 | inline void Tablet::set_cumulative_layer_point(int64_t new_point) { |
656 | | // cumulative point should only be reset to -1, or be increased |
657 | 120 | CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) |
658 | 0 | << "Unexpected cumulative point: " << new_point |
659 | 0 | << ", origin: " << _cumulative_point.load(); |
660 | 120 | _cumulative_point = new_point; |
661 | 120 | } |
662 | | |
663 | 20 | inline int64_t Tablet::cumulative_promotion_size() const { |
664 | 20 | return _cumulative_promotion_size; |
665 | 20 | } |
666 | | |
667 | 21 | inline void Tablet::set_cumulative_promotion_size(int64_t new_size) { |
668 | 21 | _cumulative_promotion_size = new_size; |
669 | 21 | } |
670 | | |
671 | | // TODO(lingbin): Why other methods that need to get information from _tablet_meta |
672 | | // are not locked, here needs a comment to explain. |
673 | 8 | inline size_t Tablet::tablet_footprint() { |
674 | 8 | std::shared_lock rdlock(_meta_lock); |
675 | 8 | return _tablet_meta->tablet_footprint(); |
676 | 8 | } |
677 | | |
678 | 0 | inline size_t Tablet::tablet_local_size() { |
679 | 0 | std::shared_lock rdlock(_meta_lock); |
680 | 0 | return _tablet_meta->tablet_local_size(); |
681 | 0 | } |
682 | | |
683 | 0 | inline size_t Tablet::tablet_remote_size() { |
684 | 0 | std::shared_lock rdlock(_meta_lock); |
685 | 0 | return _tablet_meta->tablet_remote_size(); |
686 | 0 | } |
687 | | |
688 | | // TODO(lingbin): Why other methods which need to get information from _tablet_meta |
689 | | // are not locked, here needs a comment to explain. |
690 | 16 | inline size_t Tablet::num_rows() { |
691 | 16 | std::shared_lock rdlock(_meta_lock); |
692 | 16 | return _tablet_meta->num_rows(); |
693 | 16 | } |
694 | | |
695 | 25 | inline int Tablet::version_count() const { |
696 | 25 | std::shared_lock rdlock(_meta_lock); |
697 | 25 | return _tablet_meta->version_count(); |
698 | 25 | } |
699 | | |
700 | 25 | inline int Tablet::stale_version_count() const { |
701 | 25 | std::shared_lock rdlock(_meta_lock); |
702 | 25 | return _tablet_meta->stale_version_count(); |
703 | 25 | } |
704 | | |
705 | 29 | inline Version Tablet::max_version() const { |
706 | 29 | std::shared_lock rdlock(_meta_lock); |
707 | 29 | return _tablet_meta->max_version(); |
708 | 29 | } |
709 | | |
710 | 0 | inline uint64_t Tablet::segment_count() const { |
711 | 0 | std::shared_lock rdlock(_meta_lock); |
712 | 0 | uint64_t segment_nums = 0; |
713 | 0 | for (const auto& rs_meta : _tablet_meta->all_rs_metas()) { |
714 | 0 | segment_nums += rs_meta->num_segments(); |
715 | 0 | } |
716 | 0 | return segment_nums; |
717 | 0 | } |
718 | | |
719 | 0 | inline SortType Tablet::sort_type() const { |
720 | 0 | return _tablet_meta->tablet_schema()->sort_type(); |
721 | 0 | } |
722 | | |
723 | 0 | inline size_t Tablet::sort_col_num() const { |
724 | 0 | return _tablet_meta->tablet_schema()->sort_col_num(); |
725 | 0 | } |
726 | | |
727 | 0 | inline size_t Tablet::num_columns() const { |
728 | 0 | return _tablet_meta->tablet_schema()->num_columns(); |
729 | 0 | } |
730 | | |
731 | 0 | inline size_t Tablet::num_null_columns() const { |
732 | 0 | return _tablet_meta->tablet_schema()->num_null_columns(); |
733 | 0 | } |
734 | | |
735 | 0 | inline size_t Tablet::num_short_key_columns() const { |
736 | 0 | return _tablet_meta->tablet_schema()->num_short_key_columns(); |
737 | 0 | } |
738 | | |
739 | 0 | inline size_t Tablet::num_rows_per_row_block() const { |
740 | 0 | return _tablet_meta->tablet_schema()->num_rows_per_row_block(); |
741 | 0 | } |
742 | | |
743 | 0 | inline CompressKind Tablet::compress_kind() const { |
744 | 0 | return _tablet_meta->tablet_schema()->compress_kind(); |
745 | 0 | } |
746 | | |
747 | 0 | inline double Tablet::bloom_filter_fpp() const { |
748 | 0 | return _tablet_meta->tablet_schema()->bloom_filter_fpp(); |
749 | 0 | } |
750 | | |
751 | 0 | inline size_t Tablet::next_unique_id() const { |
752 | 0 | return _tablet_meta->tablet_schema()->next_column_unique_id(); |
753 | 0 | } |
754 | | |
755 | 0 | inline size_t Tablet::row_size() const { |
756 | 0 | return _tablet_meta->tablet_schema()->row_size(); |
757 | 0 | } |
758 | | |
759 | 25 | inline int64_t Tablet::avg_rs_meta_serialize_size() const { |
760 | 25 | return _tablet_meta->avg_rs_meta_serialize_size(); |
761 | 25 | } |
762 | | |
763 | | } // namespace doris |