Coverage Report

Created: 2025-03-13 18:54

/root/doris/be/src/util/metrics.h
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <rapidjson/document.h>
21
#include <rapidjson/rapidjson.h>
22
23
#include <atomic>
24
#include <functional>
25
#include <map>
26
#include <memory>
27
#include <mutex>
28
#include <string>
29
#include <unordered_map>
30
#include <utility>
31
#include <vector>
32
33
#include "util/histogram.h"
34
35
namespace doris {
36
37
namespace rj = RAPIDJSON_NAMESPACE;
38
39
enum class MetricType { COUNTER, GAUGE, HISTOGRAM, SUMMARY, UNTYPED };
40
41
enum class MetricUnit {
42
    NANOSECONDS,
43
    MICROSECONDS,
44
    MILLISECONDS,
45
    SECONDS,
46
    BYTES,
47
    ROWS,
48
    PERCENT,
49
    REQUESTS,
50
    OPERATIONS,
51
    BLOCKS,
52
    ROWSETS,
53
    CONNECTIONS,
54
    PACKETS,
55
    NOUNIT,
56
    FILESYSTEM
57
};
58
59
std::ostream& operator<<(std::ostream& os, MetricType type);
60
const char* unit_name(MetricUnit unit);
61
62
using Labels = std::unordered_map<std::string, std::string>;
63
64
class Metric {
65
public:
66
10.5k
    Metric() = default;
67
10.3k
    virtual ~Metric() = default;
68
    virtual std::string to_string() const = 0;
69
    virtual std::string to_prometheus(const std::string& display_name, const Labels& entity_labels,
70
                                      const Labels& metric_labels) const;
71
    virtual rj::Value to_json_value(rj::Document::AllocatorType& allocator) const = 0;
72
73
private:
74
    friend class MetricRegistry;
75
};
76
77
// Metric that only can increment
78
template <typename T>
79
class AtomicMetric : public Metric {
80
public:
81
10.5k
    AtomicMetric() : _value(T()) {}
_ZN5doris12AtomicMetricIlEC2Ev
Line
Count
Source
81
9.70k
    AtomicMetric() : _value(T()) {}
_ZN5doris12AtomicMetricImEC2Ev
Line
Count
Source
81
26
    AtomicMetric() : _value(T()) {}
_ZN5doris12AtomicMetricIdEC2Ev
Line
Count
Source
81
822
    AtomicMetric() : _value(T()) {}
82
10.3k
    virtual ~AtomicMetric() = default;
_ZN5doris12AtomicMetricIlED2Ev
Line
Count
Source
82
9.53k
    virtual ~AtomicMetric() = default;
_ZN5doris12AtomicMetricImED2Ev
Line
Count
Source
82
23
    virtual ~AtomicMetric() = default;
_ZN5doris12AtomicMetricIdED2Ev
Line
Count
Source
82
808
    virtual ~AtomicMetric() = default;
83
84
628
    std::string to_string() const override { return std::to_string(value()); }
_ZNK5doris12AtomicMetricIlE9to_stringB5cxx11Ev
Line
Count
Source
84
576
    std::string to_string() const override { return std::to_string(value()); }
_ZNK5doris12AtomicMetricImE9to_stringB5cxx11Ev
Line
Count
Source
84
17
    std::string to_string() const override { return std::to_string(value()); }
_ZNK5doris12AtomicMetricIdE9to_stringB5cxx11Ev
Line
Count
Source
84
35
    std::string to_string() const override { return std::to_string(value()); }
85
86
652
    T value() const { return _value.load(); }
_ZNK5doris12AtomicMetricIlE5valueEv
Line
Count
Source
86
592
    T value() const { return _value.load(); }
_ZNK5doris12AtomicMetricImE5valueEv
Line
Count
Source
86
21
    T value() const { return _value.load(); }
_ZNK5doris12AtomicMetricIdE5valueEv
Line
Count
Source
86
39
    T value() const { return _value.load(); }
87
88
456k
    void increment(const T& delta) { _value.fetch_add(delta); }
_ZN5doris12AtomicMetricIlE9incrementERKl
Line
Count
Source
88
456k
    void increment(const T& delta) { _value.fetch_add(delta); }
_ZN5doris12AtomicMetricImE9incrementERKm
Line
Count
Source
88
3
    void increment(const T& delta) { _value.fetch_add(delta); }
_ZN5doris12AtomicMetricIdE9incrementERKd
Line
Count
Source
88
1
    void increment(const T& delta) { _value.fetch_add(delta); }
89
90
1.18k
    void set_value(const T& value) { _value.store(value); }
_ZN5doris12AtomicMetricIlE9set_valueERKl
Line
Count
Source
90
1.06k
    void set_value(const T& value) { _value.store(value); }
_ZN5doris12AtomicMetricImE9set_valueERKm
Line
Count
Source
90
115
    void set_value(const T& value) { _value.store(value); }
_ZN5doris12AtomicMetricIdE9set_valueERKd
Line
Count
Source
90
4
    void set_value(const T& value) { _value.store(value); }
91
92
6
    rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override {
93
6
        return rj::Value(value());
94
6
    }
_ZNK5doris12AtomicMetricIlE13to_json_valueERN9rapidjson19MemoryPoolAllocatorINS2_12CrtAllocatorEEE
Line
Count
Source
92
6
    rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override {
93
6
        return rj::Value(value());
94
6
    }
Unexecuted instantiation: _ZNK5doris12AtomicMetricImE13to_json_valueERN9rapidjson19MemoryPoolAllocatorINS2_12CrtAllocatorEEE
Unexecuted instantiation: _ZNK5doris12AtomicMetricIdE13to_json_valueERN9rapidjson19MemoryPoolAllocatorINS2_12CrtAllocatorEEE
95
96
protected:
97
    std::atomic<T> _value;
98
};
99
100
class HistogramMetric : public Metric {
101
public:
102
3
    HistogramMetric() = default;
103
3
    virtual ~HistogramMetric() = default;
104
105
    HistogramMetric(const HistogramMetric&) = delete;
106
    HistogramMetric& operator=(const HistogramMetric&) = delete;
107
108
    void clear();
109
    bool is_empty() const;
110
    void add(const uint64_t& value);
111
    void merge(const HistogramMetric& other);
112
    void set_histogram(const HistogramStat& stats);
113
114
0
    uint64_t min() const { return _stats.min(); }
115
0
    uint64_t max() const { return _stats.max(); }
116
0
    uint64_t num() const { return _stats.num(); }
117
0
    uint64_t sum() const { return _stats.sum(); }
118
    double median() const;
119
    double percentile(double p) const;
120
    double average() const;
121
    double standard_deviation() const;
122
    std::string to_string() const override;
123
    std::string to_prometheus(const std::string& display_name, const Labels& entity_labels,
124
                              const Labels& metric_labels) const override;
125
    rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override;
126
127
protected:
128
    static std::map<std::string, double> _s_output_percentiles;
129
    mutable std::mutex _lock;
130
    HistogramStat _stats;
131
};
132
133
template <typename T>
134
class AtomicCounter : public AtomicMetric<T> {
135
public:
136
6.13k
    AtomicCounter() = default;
_ZN5doris13AtomicCounterIlEC2Ev
Line
Count
Source
136
6.12k
    AtomicCounter() = default;
_ZN5doris13AtomicCounterImEC2Ev
Line
Count
Source
136
1
    AtomicCounter() = default;
_ZN5doris13AtomicCounterIdEC2Ev
Line
Count
Source
136
1
    AtomicCounter() = default;
137
6.03k
    virtual ~AtomicCounter() = default;
_ZN5doris13AtomicCounterIlED2Ev
Line
Count
Source
137
6.03k
    virtual ~AtomicCounter() = default;
Unexecuted instantiation: _ZN5doris13AtomicCounterImED2Ev
Unexecuted instantiation: _ZN5doris13AtomicCounterIdED2Ev
138
};
139
140
template <typename T>
141
class AtomicGauge : public AtomicMetric<T> {
142
public:
143
4.41k
    AtomicGauge() : AtomicMetric<T>() {}
_ZN5doris11AtomicGaugeIlEC2Ev
Line
Count
Source
143
3.57k
    AtomicGauge() : AtomicMetric<T>() {}
_ZN5doris11AtomicGaugeImEC2Ev
Line
Count
Source
143
25
    AtomicGauge() : AtomicMetric<T>() {}
_ZN5doris11AtomicGaugeIdEC2Ev
Line
Count
Source
143
821
    AtomicGauge() : AtomicMetric<T>() {}
144
4.33k
    virtual ~AtomicGauge() = default;
_ZN5doris11AtomicGaugeIlED2Ev
Line
Count
Source
144
3.49k
    virtual ~AtomicGauge() = default;
_ZN5doris11AtomicGaugeImED2Ev
Line
Count
Source
144
23
    virtual ~AtomicGauge() = default;
_ZN5doris11AtomicGaugeIdED2Ev
Line
Count
Source
144
808
    virtual ~AtomicGauge() = default;
145
};
146
147
using IntCounter = AtomicCounter<int64_t>;
148
using UIntCounter = AtomicCounter<uint64_t>;
149
using DoubleCounter = AtomicCounter<double>;
150
using IntGauge = AtomicGauge<int64_t>;
151
using UIntGauge = AtomicGauge<uint64_t>;
152
using DoubleGauge = AtomicGauge<double>;
153
using Labels = std::unordered_map<std::string, std::string>;
154
155
struct MetricPrototype {
156
public:
157
    MetricPrototype(MetricType type_, MetricUnit unit_, std::string name_,
158
                    std::string description_ = "", std::string group_name_ = "",
159
                    Labels labels_ = Labels(), bool is_core_metric_ = false)
160
            : is_core_metric(is_core_metric_),
161
              type(type_),
162
              unit(unit_),
163
              name(std::move(name_)),
164
              description(std::move(description_)),
165
              group_name(std::move(group_name_)),
166
435
              labels(std::move(labels_)) {}
167
168
    std::string simple_name() const;
169
    std::string combine_name(const std::string& registry_name) const;
170
    std::string to_prometheus(const std::string& registry_name) const;
171
172
    bool is_core_metric;
173
    MetricType type;
174
    MetricUnit unit;
175
    std::string name;
176
    std::string description;
177
    std::string group_name;
178
    Labels labels;
179
};
180
181
#define DEFINE_METRIC_PROTOTYPE(name, type, unit, desc, group, labels, core) \
182
    ::doris::MetricPrototype METRIC_##name(type, unit, #name, desc, group, labels, core)
183
184
#define DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(name, unit) \
185
    DEFINE_METRIC_PROTOTYPE(name, MetricType::COUNTER, unit, "", "", Labels(), false)
186
187
#define DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(name, unit, desc) \
188
    DEFINE_METRIC_PROTOTYPE(name, MetricType::COUNTER, unit, desc, "", Labels(), false)
189
190
#define DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(name, unit, desc, group, labels) \
191
    DEFINE_METRIC_PROTOTYPE(name, MetricType::COUNTER, unit, desc, #group, labels, false)
192
193
#define DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(name, unit) \
194
    DEFINE_METRIC_PROTOTYPE(name, MetricType::GAUGE, unit, "", "", Labels(), false)
195
196
#define DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(name, unit) \
197
    DEFINE_METRIC_PROTOTYPE(name, MetricType::GAUGE, unit, "", "", Labels(), true)
198
199
#define DEFINE_GAUGE_METRIC_PROTOTYPE_3ARG(name, unit, desc) \
200
    DEFINE_METRIC_PROTOTYPE(name, MetricType::GAUGE, unit, desc, "", Labels(), false)
201
202
#define DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(name, unit, desc, group, labels) \
203
    DEFINE_METRIC_PROTOTYPE(name, MetricType::GAUGE, unit, desc, #group, labels, false)
204
205
#define DEFINE_HISTOGRAM_METRIC_PROTOTYPE_2ARG(name, unit) \
206
    DEFINE_METRIC_PROTOTYPE(name, MetricType::HISTOGRAM, unit, "", "", Labels(), false)
207
208
#define INT_COUNTER_METRIC_REGISTER(entity, metric) \
209
6.36k
    metric = (IntCounter*)(entity->register_metric<IntCounter>(&METRIC_##metric))
210
211
#define INT_GAUGE_METRIC_REGISTER(entity, metric) \
212
3.94k
    metric = (IntGauge*)(entity->register_metric<IntGauge>(&METRIC_##metric))
213
214
#define DOUBLE_GAUGE_METRIC_REGISTER(entity, metric) \
215
906
    metric = (DoubleGauge*)(entity->register_metric<DoubleGauge>(&METRIC_##metric))
216
217
#define INT_UGAUGE_METRIC_REGISTER(entity, metric) \
218
10
    metric = (UIntGauge*)(entity->register_metric<UIntGauge>(&METRIC_##metric))
219
220
#define HISTOGRAM_METRIC_REGISTER(entity, metric) \
221
1
    metric = (HistogramMetric*)(entity->register_metric<HistogramMetric>(&METRIC_##metric))
222
223
#define METRIC_DEREGISTER(entity, metric) entity->deregister_metric(&METRIC_##metric)
224
225
// For 'metrics' in MetricEntity.
226
struct MetricPrototypeHash {
227
12.5k
    size_t operator()(const MetricPrototype* metric_prototype) const {
228
12.5k
        return std::hash<std::string>()(metric_prototype->group_name.empty()
229
12.5k
                                                ? metric_prototype->name
230
12.5k
                                                : metric_prototype->group_name);
231
12.5k
    }
232
};
233
234
struct MetricPrototypeEqualTo {
235
5.65k
    bool operator()(const MetricPrototype* first, const MetricPrototype* second) const {
236
5.65k
        return first->group_name == second->group_name && first->name == second->name;
237
5.65k
    }
238
};
239
240
using MetricMap = std::unordered_map<const MetricPrototype*, Metric*, MetricPrototypeHash,
241
                                     MetricPrototypeEqualTo>;
242
243
enum class MetricEntityType { kServer, kTablet };
244
245
class MetricEntity {
246
public:
247
    MetricEntity(MetricEntityType type, std::string name, Labels labels)
248
1.57k
            : _type(type), _name(std::move(name)), _labels(std::move(labels)) {}
249
1.55k
    ~MetricEntity() {
250
10.3k
        for (auto& metric : _metrics) {
251
10.3k
            delete metric.second;
252
10.3k
        }
253
1.55k
    }
254
255
3.16k
    const std::string& name() const { return _name; }
256
257
    template <typename T>
258
11.5k
    Metric* register_metric(const MetricPrototype* metric_type) {
259
11.5k
        std::lock_guard<std::mutex> l(_lock);
260
11.5k
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
11.5k
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
10.5k
            inserted_metric.first->second = new T();
264
10.5k
        }
265
11.5k
        return inserted_metric.first->second;
266
11.5k
    }
_ZN5doris12MetricEntity15register_metricINS_13AtomicCounterIlEEEEPNS_6MetricEPKNS_15MetricPrototypeE
Line
Count
Source
258
6.45k
    Metric* register_metric(const MetricPrototype* metric_type) {
259
6.45k
        std::lock_guard<std::mutex> l(_lock);
260
6.45k
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
6.45k
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
6.12k
            inserted_metric.first->second = new T();
264
6.12k
        }
265
6.45k
        return inserted_metric.first->second;
266
6.45k
    }
_ZN5doris12MetricEntity15register_metricINS_11AtomicGaugeIlEEEEPNS_6MetricEPKNS_15MetricPrototypeE
Line
Count
Source
258
3.94k
    Metric* register_metric(const MetricPrototype* metric_type) {
259
3.94k
        std::lock_guard<std::mutex> l(_lock);
260
3.94k
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
3.94k
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
3.57k
            inserted_metric.first->second = new T();
264
3.57k
        }
265
3.94k
        return inserted_metric.first->second;
266
3.94k
    }
_ZN5doris12MetricEntity15register_metricINS_15HistogramMetricEEEPNS_6MetricEPKNS_15MetricPrototypeE
Line
Count
Source
258
3
    Metric* register_metric(const MetricPrototype* metric_type) {
259
3
        std::lock_guard<std::mutex> l(_lock);
260
3
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
3
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
3
            inserted_metric.first->second = new T();
264
3
        }
265
3
        return inserted_metric.first->second;
266
3
    }
_ZN5doris12MetricEntity15register_metricINS_11AtomicGaugeIdEEEEPNS_6MetricEPKNS_15MetricPrototypeE
Line
Count
Source
258
906
    Metric* register_metric(const MetricPrototype* metric_type) {
259
906
        std::lock_guard<std::mutex> l(_lock);
260
906
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
906
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
820
            inserted_metric.first->second = new T();
264
820
        }
265
906
        return inserted_metric.first->second;
266
906
    }
_ZN5doris12MetricEntity15register_metricINS_11AtomicGaugeImEEEEPNS_6MetricEPKNS_15MetricPrototypeE
Line
Count
Source
258
232
    Metric* register_metric(const MetricPrototype* metric_type) {
259
232
        std::lock_guard<std::mutex> l(_lock);
260
232
        auto inserted_metric = _metrics.insert(std::make_pair(metric_type, nullptr));
261
232
        if (inserted_metric.second) {
262
            // If not exist, make a new metric pointer
263
24
            inserted_metric.first->second = new T();
264
24
        }
265
232
        return inserted_metric.first->second;
266
232
    }
267
268
    void deregister_metric(const MetricPrototype* metric_type);
269
    Metric* get_metric(const std::string& name, const std::string& group_name = "") const;
270
271
    // Register a hook, this hook will called before get_metric is called
272
    void register_hook(const std::string& name, const std::function<void()>& hook);
273
    void deregister_hook(const std::string& name);
274
    void trigger_hook_unlocked(bool force) const;
275
276
private:
277
    friend class MetricRegistry;
278
    friend struct MetricEntityHash;
279
    friend struct MetricEntityEqualTo;
280
281
    MetricEntityType _type;
282
    std::string _name;
283
    Labels _labels;
284
285
    mutable std::mutex _lock;
286
    MetricMap _metrics;
287
    std::map<std::string, std::function<void()>> _hooks;
288
};
289
290
struct MetricEntityHash {
291
3.16k
    size_t operator()(const std::shared_ptr<MetricEntity> metric_entity) const {
292
3.16k
        return std::hash<std::string>()(metric_entity->name());
293
3.16k
    }
294
};
295
296
struct MetricEntityEqualTo {
297
    bool operator()(const std::shared_ptr<MetricEntity> first,
298
1.64k
                    const std::shared_ptr<MetricEntity> second) const {
299
1.64k
        return first->_type == second->_type && first->_name == second->_name &&
300
1.64k
               first->_labels == second->_labels;
301
1.64k
    }
302
};
303
304
using EntityMetricsByType =
305
        std::unordered_map<const MetricPrototype*, std::vector<std::pair<MetricEntity*, Metric*>>,
306
                           MetricPrototypeHash, MetricPrototypeEqualTo>;
307
308
class MetricRegistry {
309
public:
310
6
    MetricRegistry(std::string name) : _name(std::move(name)) {}
311
    ~MetricRegistry();
312
313
    std::shared_ptr<MetricEntity> register_entity(
314
            const std::string& name, const Labels& labels = {},
315
            MetricEntityType type = MetricEntityType::kServer);
316
    void deregister_entity(const std::shared_ptr<MetricEntity>& entity);
317
    std::shared_ptr<MetricEntity> get_entity(const std::string& name, const Labels& labels = {},
318
                                             MetricEntityType type = MetricEntityType::kServer);
319
320
    void trigger_all_hooks(bool force) const;
321
322
    std::string to_prometheus(bool with_tablet_metrics = false) const;
323
    std::string to_json(bool with_tablet_metrics = false) const;
324
    std::string to_core_string() const;
325
326
private:
327
    const std::string _name;
328
329
    mutable std::mutex _lock;
330
    // MetricEntity -> register count
331
    std::unordered_map<std::shared_ptr<MetricEntity>, int32_t, MetricEntityHash,
332
                       MetricEntityEqualTo>
333
            _entities;
334
};
335
336
} // namespace doris