Coverage Report

Created: 2026-04-18 09:33

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/cloud/cloud_tablet_hotspot.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <array>
21
#include <atomic>
22
#include <chrono>
23
#include <condition_variable>
24
#include <cstdint>
25
#include <deque>
26
#include <memory>
27
#include <mutex>
28
#include <thread>
29
#include <unordered_map>
30
#include <vector>
31
32
#include "gen_cpp/BackendService.h"
33
#include "storage/tablet/tablet_fwd.h"
34
35
namespace doris {
36
37
// This counter is used to count the tablet query frequency
38
struct HotspotCounter {
39
    HotspotCounter(int64_t table_id, int64_t index_id, int64_t partition_id)
40
89.7k
            : table_id(table_id), index_id(index_id), partition_id(partition_id) {}
41
    // One break point every hour
42
    void make_dot_point();
43
    uint64_t qpd();
44
    uint64_t qpw();
45
    int64_t table_id;
46
    int64_t index_id;
47
    int64_t partition_id;
48
    std::chrono::system_clock::time_point last_access_time;
49
    std::atomic_uint64_t cur_counter {0};
50
    std::deque<uint64_t> history_counters;
51
    std::atomic_uint64_t week_history_counter {0};
52
    std::atomic_uint64_t day_history_counter {0};
53
    static inline int64_t time_interval = 1 * 60 * 60;
54
    static inline int64_t week_counters_size = ((7 * 24 * 60 * 60) / time_interval) - 1;
55
    static inline int64_t day_counters_size = ((24 * 60 * 60) / time_interval) - 1;
56
};
57
58
using HotspotCounterPtr = std::shared_ptr<HotspotCounter>;
59
using TabletHotspotMapKey = std::pair<int64_t, int64_t>;
60
61
struct TabletHotspotMapValue {
62
    uint64_t qpd = 0; // query per day
63
    uint64_t qpw = 0; // query per week
64
    int64_t last_access_time;
65
};
66
67
struct MapKeyHash {
68
4
    int64_t operator()(const std::pair<int64_t, int64_t>& key) const {
69
4
        return std::hash<int64_t> {}(key.first) + std::hash<int64_t> {}(key.second);
70
4
    }
71
};
72
73
class TabletHotspot {
74
public:
75
    struct MaintenanceStats {
76
        uint64_t total_counters_before_gc = 0;
77
        uint64_t total_counters_after_gc = 0;
78
        uint64_t non_empty_slots_before_gc = 0;
79
        uint64_t non_empty_slots_after_gc = 0;
80
        uint64_t max_slot_size_before_gc = 0;
81
        uint64_t max_slot_size_after_gc = 0;
82
        uint64_t sum_bucket_count_before_gc = 0;
83
        uint64_t sum_bucket_count_after_gc = 0;
84
        uint64_t copied_counters = 0;
85
        uint64_t evicted_counters = 0;
86
        uint64_t compacted_slots = 0;
87
        uint64_t gc_elapsed_ms = 0;
88
    };
89
90
    explicit TabletHotspot(bool start_counter_thread = true);
91
    ~TabletHotspot();
92
    // When query the tablet, count it
93
    void count(const BaseTablet& tablet);
94
    void get_top_n_hot_partition(std::vector<THotTableMessage>* hot_tables);
95
96
private:
97
    void count(int64_t tablet_id, int64_t table_id, int64_t index_id, int64_t partition_id);
98
    void make_dot_point();
99
    MaintenanceStats run_maintenance_once(std::chrono::system_clock::time_point now);
100
    static bool is_gc_eligible(const HotspotCounter& counter,
101
                               std::chrono::system_clock::time_point now);
102
103
    struct HotspotMap {
104
        std::mutex mtx;
105
        std::unordered_map<int64_t, HotspotCounterPtr> map;
106
    };
107
    static constexpr size_t s_slot_size = 1024;
108
    std::array<HotspotMap, s_slot_size> _tablets_hotspot;
109
    std::thread _counter_thread;
110
    bool _closed {false};
111
    std::mutex _mtx;
112
    std::condition_variable _cond;
113
114
    std::mutex _last_partitions_mtx;
115
    std::unordered_map<TabletHotspotMapKey, std::unordered_map<int64_t, TabletHotspotMapValue>,
116
                       MapKeyHash>
117
            _last_day_hot_partitions;
118
    std::unordered_map<TabletHotspotMapKey, std::unordered_map<int64_t, TabletHotspotMapValue>,
119
                       MapKeyHash>
120
            _last_week_hot_partitions;
121
    std::atomic_uint64_t _count_calls_total {0};
122
    std::atomic_uint64_t _existing_hit_total {0};
123
    std::atomic_uint64_t _new_counter_total {0};
124
    std::atomic_uint64_t _last_round_new_counter_total {0};
125
    std::atomic_uint64_t _get_top_n_hot_partition_call_total {0};
126
    std::atomic_uint64_t _make_dot_point_round_total {0};
127
};
128
129
} // namespace doris