/root/doris/be/src/util/runtime_profile.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/apache/impala/blob/branch-2.9.0/be/src/util/runtime-profile.h |
19 | | // and modified by Doris |
20 | | |
21 | | #pragma once |
22 | | |
23 | | #include <gen_cpp/Metrics_types.h> |
24 | | #include <glog/logging.h> |
25 | | #include <stdint.h> |
26 | | |
27 | | #include <algorithm> |
28 | | #include <atomic> |
29 | | #include <cstdint> |
30 | | #include <functional> |
31 | | #include <iostream> |
32 | | #include <map> |
33 | | #include <memory> |
34 | | #include <mutex> |
35 | | #include <set> |
36 | | #include <string> |
37 | | #include <utility> |
38 | | #include <vector> |
39 | | |
40 | | #include "common/compiler_util.h" // IWYU pragma: keep |
41 | | #include "util/binary_cast.hpp" |
42 | | #include "util/container_util.hpp" |
43 | | #include "util/pretty_printer.h" |
44 | | #include "util/stopwatch.hpp" |
45 | | |
46 | | namespace doris { |
47 | | class TRuntimeProfileNode; |
48 | | class TRuntimeProfileTree; |
49 | | |
50 | | // Some macro magic to generate unique ids using __COUNTER__ |
51 | 260k | #define CONCAT_IMPL(x, y) x##y |
52 | 260k | #define MACRO_CONCAT(x, y) CONCAT_IMPL(x, y) |
53 | | |
54 | 0 | #define ADD_LABEL_COUNTER(profile, name) (profile)->add_counter(name, TUnit::NONE) |
55 | | #define ADD_LABEL_COUNTER_WITH_LEVEL(profile, name, level) \ |
56 | 0 | (profile)->add_counter_with_level(name, TUnit::NONE, level) |
57 | 2.82k | #define ADD_COUNTER(profile, name, type) (profile)->add_counter(name, type) |
58 | | #define ADD_COUNTER_WITH_LEVEL(profile, name, type, level) \ |
59 | 20 | (profile)->add_counter_with_level(name, type, level) |
60 | 911 | #define ADD_TIMER(profile, name) (profile)->add_counter(name, TUnit::TIME_NS) |
61 | | #define ADD_TIMER_WITH_LEVEL(profile, name, level) \ |
62 | 50 | (profile)->add_counter_with_level(name, TUnit::TIME_NS, level) |
63 | 0 | #define ADD_CHILD_COUNTER(profile, name, type, parent) (profile)->add_counter(name, type, parent) |
64 | | #define ADD_CHILD_COUNTER_WITH_LEVEL(profile, name, type, parent, level) \ |
65 | 14 | (profile)->add_counter(name, type, parent, level) |
66 | 600 | #define ADD_CHILD_TIMER(profile, name, parent) (profile)->add_counter(name, TUnit::TIME_NS, parent) |
67 | | #define ADD_CHILD_TIMER_WITH_LEVEL(profile, name, parent, level) \ |
68 | 16 | (profile)->add_counter(name, TUnit::TIME_NS, parent, level) |
69 | 966 | #define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c) |
70 | | #define SCOPED_TIMER_ATOMIC(c) \ |
71 | | ScopedTimer<MonotonicStopWatch, std::atomic_bool> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c) |
72 | | #define SCOPED_CPU_TIMER(c) \ |
73 | 0 | ScopedTimer<ThreadCpuStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c) |
74 | | #define CANCEL_SAFE_SCOPED_TIMER(c, is_cancelled) \ |
75 | | ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c, is_cancelled) |
76 | | #define SCOPED_RAW_TIMER(c) \ |
77 | 259k | doris::ScopedRawTimer<doris::MonotonicStopWatch, int64_t> MACRO_CONCAT(SCOPED_RAW_TIMER, \ |
78 | 259k | __COUNTER__)(c) |
79 | | #define SCOPED_ATOMIC_TIMER(c) \ |
80 | 0 | ScopedRawTimer<MonotonicStopWatch, std::atomic<int64_t>> MACRO_CONCAT(SCOPED_ATOMIC_TIMER, \ |
81 | 0 | __COUNTER__)(c) |
82 | 3.50k | #define COUNTER_UPDATE(c, v) (c)->update(v) |
83 | 233 | #define COUNTER_SET(c, v) (c)->set(v) |
84 | | |
85 | | class ObjectPool; |
86 | | |
87 | | // Runtime profile is a group of profiling counters. It supports adding named counters |
88 | | // and being able to serialize and deserialize them. |
89 | | // The profiles support a tree structure to form a hierarchy of counters. |
90 | | // Runtime profiles supports measuring wall clock rate based counters. There is a |
91 | | // single thread per process that will convert an amount (i.e. bytes) counter to a |
92 | | // corresponding rate based counter. This thread wakes up at fixed intervals and updates |
93 | | // all of the rate counters. |
94 | | // Thread-safe. |
95 | | class RuntimeProfile { |
96 | | public: |
97 | | class Counter { |
98 | | public: |
99 | | Counter(TUnit::type type, int64_t value = 0, int64_t level = 3) |
100 | 30.9k | : _value(value), _type(type), _level(level) {} |
101 | 30.8k | virtual ~Counter() = default; |
102 | | |
103 | 0 | virtual Counter* clone() const { return new Counter(type(), value(), _level); } |
104 | | |
105 | 4.42k | virtual void update(int64_t delta) { _value.fetch_add(delta, std::memory_order_relaxed); } |
106 | | |
107 | 0 | void bit_or(int64_t delta) { _value.fetch_or(delta, std::memory_order_relaxed); } |
108 | | |
109 | 233 | virtual void set(int64_t value) { _value.store(value, std::memory_order_relaxed); } |
110 | | |
111 | 0 | virtual void set(double value) { |
112 | 0 | DCHECK_EQ(sizeof(value), sizeof(int64_t)); |
113 | 0 | _value.store(binary_cast<double, int64_t>(value), std::memory_order_relaxed); |
114 | 0 | } |
115 | | |
116 | 30 | virtual int64_t value() const { return _value.load(std::memory_order_relaxed); } |
117 | | |
118 | 0 | virtual double double_value() const { |
119 | 0 | return binary_cast<int64_t, double>(_value.load(std::memory_order_relaxed)); |
120 | 0 | } |
121 | | |
122 | | virtual void to_thrift(const std::string& name, std::vector<TCounter>& tcounters, |
123 | 0 | std::map<std::string, std::set<std::string>>& child_counters_map) { |
124 | 0 | TCounter counter; |
125 | 0 | counter.name = name; |
126 | 0 | counter.value = this->value(); |
127 | 0 | counter.type = this->type(); |
128 | 0 | counter.__set_level(this->level()); |
129 | 0 | tcounters.push_back(std::move(counter)); |
130 | 0 | } |
131 | | |
132 | | virtual void pretty_print(std::ostream* s, const std::string& prefix, |
133 | 0 | const std::string& name) const { |
134 | 0 | std::ostream& stream = *s; |
135 | 0 | stream << prefix << " - " << name << ": " |
136 | 0 | << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type()) |
137 | 0 | << std::endl; |
138 | 0 | } |
139 | | |
140 | 929 | TUnit::type type() const { return _type; } |
141 | | |
142 | 0 | virtual int64_t level() const { return _level; } |
143 | | |
144 | | private: |
145 | | friend class RuntimeProfile; |
146 | | |
147 | | std::atomic<int64_t> _value; |
148 | | TUnit::type _type; |
149 | | int64_t _level; |
150 | | }; |
151 | | |
152 | | /// A counter that keeps track of the highest value seen (reporting that |
153 | | /// as value()) and the current value. |
154 | | class HighWaterMarkCounter : public Counter { |
155 | | public: |
156 | | HighWaterMarkCounter(TUnit::type unit, int64_t level, const std::string& parent_name, |
157 | | int64_t value = 0, int64_t current_value = 0) |
158 | | : Counter(unit, value, level), |
159 | | current_value_(current_value), |
160 | 26 | _parent_name(parent_name) {} |
161 | | |
162 | 0 | virtual Counter* clone() const override { |
163 | 0 | return new HighWaterMarkCounter(type(), level(), parent_name(), value(), |
164 | 0 | current_value()); |
165 | 0 | } |
166 | | |
167 | 0 | void add(int64_t delta) { |
168 | 0 | current_value_.fetch_add(delta, std::memory_order_relaxed); |
169 | 0 | if (delta > 0) { |
170 | 0 | UpdateMax(current_value_); |
171 | 0 | } |
172 | 0 | } |
173 | 0 | virtual void update(int64_t delta) override { add(delta); } |
174 | | |
175 | | virtual void to_thrift( |
176 | | const std::string& name, std::vector<TCounter>& tcounters, |
177 | 0 | std::map<std::string, std::set<std::string>>& child_counters_map) override { |
178 | 0 | { |
179 | 0 | TCounter counter; |
180 | 0 | counter.name = name; |
181 | 0 | counter.value = this->current_value(); |
182 | 0 | counter.type = this->type(); |
183 | 0 | counter.__set_level(this->level()); |
184 | 0 | tcounters.push_back(std::move(counter)); |
185 | 0 | } |
186 | 0 | { |
187 | 0 | TCounter counter; |
188 | 0 | std::string peak_name = name + "Peak"; |
189 | 0 | counter.name = peak_name; |
190 | 0 | counter.value = this->value(); |
191 | 0 | counter.type = this->type(); |
192 | 0 | counter.__set_level(this->level()); |
193 | 0 | tcounters.push_back(std::move(counter)); |
194 | 0 | child_counters_map[_parent_name].insert(peak_name); |
195 | 0 | } |
196 | 0 | } |
197 | | |
198 | | virtual void pretty_print(std::ostream* s, const std::string& prefix, |
199 | 0 | const std::string& name) const override { |
200 | 0 | std::ostream& stream = *s; |
201 | 0 | stream << prefix << " - " << name |
202 | 0 | << " Current: " << PrettyPrinter::print(current_value(), type()) << " (Peak: " |
203 | 0 | << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type()) << ")" |
204 | 0 | << std::endl; |
205 | 0 | } |
206 | | |
207 | | /// Tries to increase the current value by delta. If current_value() + delta |
208 | | /// exceeds max, return false and current_value is not changed. |
209 | 0 | bool try_add(int64_t delta, int64_t max) { |
210 | 0 | while (true) { |
211 | 0 | int64_t old_val = current_value_.load(std::memory_order_relaxed); |
212 | 0 | int64_t new_val = old_val + delta; |
213 | 0 | if (UNLIKELY(new_val > max)) return false; |
214 | 0 | if (LIKELY(current_value_.compare_exchange_weak(old_val, new_val, |
215 | 0 | std::memory_order_relaxed))) { |
216 | 0 | UpdateMax(new_val); |
217 | 0 | return true; |
218 | 0 | } |
219 | 0 | } |
220 | 0 | } |
221 | | |
222 | 0 | void set(int64_t v) override { |
223 | 0 | current_value_.store(v, std::memory_order_relaxed); |
224 | 0 | UpdateMax(v); |
225 | 0 | } |
226 | | |
227 | 0 | int64_t current_value() const { return current_value_.load(std::memory_order_relaxed); } |
228 | | |
229 | 0 | std::string parent_name() const { return _parent_name; } |
230 | | |
231 | | private: |
232 | | /// Set '_value' to 'v' if 'v' is larger than '_value'. The entire operation is |
233 | | /// atomic. |
234 | 0 | void UpdateMax(int64_t v) { |
235 | 0 | while (true) { |
236 | 0 | int64_t old_max = _value.load(std::memory_order_relaxed); |
237 | 0 | int64_t new_max = std::max(old_max, v); |
238 | 0 | if (new_max == old_max) { |
239 | 0 | break; // Avoid atomic update. |
240 | 0 | } |
241 | 0 | if (LIKELY(_value.compare_exchange_weak(old_max, new_max, |
242 | 0 | std::memory_order_relaxed))) { |
243 | 0 | break; |
244 | 0 | } |
245 | 0 | } |
246 | 0 | } |
247 | | |
248 | | /// The current value of the counter. _value in the super class represents |
249 | | /// the high water mark. |
250 | | std::atomic<int64_t> current_value_; |
251 | | |
252 | | const std::string _parent_name; |
253 | | }; |
254 | | |
255 | | using DerivedCounterFunction = std::function<int64_t()>; |
256 | | |
257 | | // A DerivedCounter also has a name and type, but the value is computed. |
258 | | // Do not call Set() and Update(). |
259 | | class DerivedCounter : public Counter { |
260 | | public: |
261 | | DerivedCounter(TUnit::type type, const DerivedCounterFunction& counter_fn, |
262 | | int64_t value = 0, int64_t level = 1) |
263 | 0 | : Counter(type, value, level), _counter_fn(counter_fn) {} |
264 | | |
265 | 0 | virtual Counter* clone() const override { |
266 | 0 | return new DerivedCounter(type(), _counter_fn, value(), level()); |
267 | 0 | } |
268 | | |
269 | 0 | int64_t value() const override { return _counter_fn(); } |
270 | | |
271 | | private: |
272 | | DerivedCounterFunction _counter_fn; |
273 | | }; |
274 | | |
275 | | // NonZeroCounter will not be converted to Thrift if the value is 0. |
276 | | class NonZeroCounter : public Counter { |
277 | | public: |
278 | | NonZeroCounter(TUnit::type type, int64_t level, const std::string& parent_name, |
279 | | int64_t value = 0) |
280 | 0 | : Counter(type, value, level), _parent_name(parent_name) {} |
281 | | |
282 | 0 | virtual Counter* clone() const override { |
283 | 0 | return new NonZeroCounter(type(), level(), parent_name(), value()); |
284 | 0 | } |
285 | | |
286 | | void to_thrift(const std::string& name, std::vector<TCounter>& tcounters, |
287 | 0 | std::map<std::string, std::set<std::string>>& child_counters_map) override { |
288 | 0 | if (this->_value > 0) { |
289 | 0 | Counter::to_thrift(name, tcounters, child_counters_map); |
290 | 0 | } else { |
291 | | // remove it |
292 | 0 | child_counters_map[_parent_name].erase(name); |
293 | 0 | } |
294 | 0 | } |
295 | | |
296 | 0 | std::string parent_name() const { return _parent_name; } |
297 | | |
298 | | private: |
299 | | const std::string _parent_name; |
300 | | }; |
301 | | |
302 | | // An EventSequence captures a sequence of events (each added by |
303 | | // calling MarkEvent). Each event has a text label, and a time |
304 | | // (measured relative to the moment start() was called as t=0). It is |
305 | | // useful for tracking the evolution of some serial process, such as |
306 | | // the query lifecycle. |
307 | | // Not thread-safe. |
308 | | class EventSequence { |
309 | | public: |
310 | 0 | EventSequence() = default; |
311 | | |
312 | | // starts the timer without resetting it. |
313 | 0 | void start() { _sw.start(); } |
314 | | |
315 | | // stops (or effectively pauses) the timer. |
316 | 0 | void stop() { _sw.stop(); } |
317 | | |
318 | | // Stores an event in sequence with the given label and the |
319 | | // current time (relative to the first time start() was called) as |
320 | | // the timestamp. |
321 | 0 | void mark_event(const std::string& label) { |
322 | 0 | _events.push_back(make_pair(label, _sw.elapsed_time())); |
323 | 0 | } |
324 | | |
325 | 0 | int64_t elapsed_time() { return _sw.elapsed_time(); } |
326 | | |
327 | | // An Event is a <label, timestamp> pair |
328 | | using Event = std::pair<std::string, int64_t>; |
329 | | |
330 | | // An EventList is a sequence of Events, in increasing timestamp order |
331 | | using EventList = std::vector<Event>; |
332 | | |
333 | 0 | const EventList& events() const { return _events; } |
334 | | |
335 | | private: |
336 | | // Stored in increasing time order |
337 | | EventList _events; |
338 | | |
339 | | // Timer which allows events to be timestamped when they are recorded. |
340 | | MonotonicStopWatch _sw; |
341 | | }; |
342 | | |
343 | | // Create a runtime profile object with 'name'. |
344 | | RuntimeProfile(const std::string& name, bool is_averaged_profile = false); |
345 | | |
346 | | ~RuntimeProfile(); |
347 | | |
348 | | // Adds a child profile. This is thread safe. |
349 | | // 'indent' indicates whether the child will be printed w/ extra indentation |
350 | | // relative to the parent. |
351 | | // If location is non-null, child will be inserted after location. Location must |
352 | | // already be added to the profile. |
353 | | void add_child(RuntimeProfile* child, bool indent, RuntimeProfile* location); |
354 | | |
355 | | void insert_child_head(RuntimeProfile* child, bool indent); |
356 | | |
357 | | void add_child_unlock(RuntimeProfile* child, bool indent, RuntimeProfile* loc); |
358 | | |
359 | | /// Creates a new child profile with the given 'name'. A child profile with that name |
360 | | /// must not already exist. If 'prepend' is true, prepended before other child profiles, |
361 | | /// otherwise appended after other child profiles. |
362 | | RuntimeProfile* create_child(const std::string& name, bool indent = true, bool prepend = false); |
363 | | |
364 | | // Merges the src profile into this one, combining counters that have an identical |
365 | | // path. Info strings from profiles are not merged. 'src' would be a const if it |
366 | | // weren't for locking. |
367 | | // Calling this concurrently on two RuntimeProfiles in reverse order results in |
368 | | // undefined behavior. |
369 | | void merge(RuntimeProfile* src); |
370 | | |
371 | | // Updates this profile w/ the thrift profile: behaves like Merge(), except |
372 | | // that existing counters are updated rather than added up. |
373 | | // Info strings matched up by key and are updated or added, depending on whether |
374 | | // the key has already been registered. |
375 | | void update(const TRuntimeProfileTree& thrift_profile); |
376 | | |
377 | | // Add a counter with 'name'/'type'. Returns a counter object that the caller can |
378 | | // update. The counter is owned by the RuntimeProfile object. |
379 | | // If parent_counter_name is a non-empty string, the counter is added as a child of |
380 | | // parent_counter_name. |
381 | | // If the counter already exists, the existing counter object is returned. |
382 | | Counter* add_counter(const std::string& name, TUnit::type type, |
383 | | const std::string& parent_counter_name, int64_t level = 2); |
384 | 4.72k | Counter* add_counter(const std::string& name, TUnit::type type) { |
385 | 4.72k | return add_counter(name, type, ""); |
386 | 4.72k | } |
387 | | |
388 | 70 | Counter* add_counter_with_level(const std::string& name, TUnit::type type, int64_t level) { |
389 | 70 | return add_counter(name, type, "", level); |
390 | 70 | } |
391 | | |
392 | | NonZeroCounter* add_nonzero_counter(const std::string& name, TUnit::type type, |
393 | | const std::string& parent_counter_name = "", |
394 | | int64_t level = 2); |
395 | | |
396 | | // Add a derived counter with 'name'/'type'. The counter is owned by the |
397 | | // RuntimeProfile object. |
398 | | // If parent_counter_name is a non-empty string, the counter is added as a child of |
399 | | // parent_counter_name. |
400 | | // Returns nullptr if the counter already exists. |
401 | | DerivedCounter* add_derived_counter(const std::string& name, TUnit::type type, |
402 | | const DerivedCounterFunction& counter_fn, |
403 | | const std::string& parent_counter_name); |
404 | | |
405 | | // Gets the counter object with 'name'. Returns nullptr if there is no counter with |
406 | | // that name. |
407 | | Counter* get_counter(const std::string& name); |
408 | | |
409 | | // Adds all counters with 'name' that are registered either in this or |
410 | | // in any of the child profiles to 'counters'. |
411 | | void get_counters(const std::string& name, std::vector<Counter*>* counters); |
412 | | |
413 | | // Helper to append to the "ExecOption" info string. |
414 | 0 | void append_exec_option(const std::string& option) { add_info_string("ExecOption", option); } |
415 | | |
416 | | // Adds a string to the runtime profile. If a value already exists for 'key', |
417 | | // the value will be updated. |
418 | | void add_info_string(const std::string& key, const std::string& value); |
419 | | |
420 | | // Creates and returns a new EventSequence (owned by the runtime |
421 | | // profile) - unless a timer with the same 'key' already exists, in |
422 | | // which case it is returned. |
423 | | // TODO: EventSequences are not merged by Merge() |
424 | | EventSequence* add_event_sequence(const std::string& key); |
425 | | |
426 | | // Returns a pointer to the info string value for 'key'. Returns nullptr if |
427 | | // the key does not exist. |
428 | | const std::string* get_info_string(const std::string& key); |
429 | | |
430 | | // Returns the counter for the total elapsed time. |
431 | 0 | Counter* total_time_counter() { return &_counter_total_time; } |
432 | | |
433 | | // Prints the counters in a name: value format. |
434 | | // Does not hold locks when it makes any function calls. |
435 | | void pretty_print(std::ostream* s, const std::string& prefix = "") const; |
436 | | |
437 | | // Serializes profile to thrift. |
438 | | // Does not hold locks when it makes any function calls. |
439 | | void to_thrift(TRuntimeProfileTree* tree); |
440 | | void to_thrift(std::vector<TRuntimeProfileNode>* nodes); |
441 | | |
442 | | // Divides all counters by n |
443 | | void divide(int n); |
444 | | |
445 | | void get_children(std::vector<RuntimeProfile*>* children); |
446 | | |
447 | | // Gets all profiles in tree, including this one. |
448 | | void get_all_children(std::vector<RuntimeProfile*>* children); |
449 | | |
450 | | // Returns the number of counters in this profile |
451 | 0 | int num_counters() const { return _counter_map.size(); } |
452 | | |
453 | | // Returns name of this profile |
454 | 0 | const std::string& name() const { return _name; } |
455 | | |
456 | | // *only call this on top-level profiles* |
457 | | // (because it doesn't re-file child profiles) |
458 | 0 | void set_name(const std::string& name) { _name = name; } |
459 | | |
460 | 0 | int64_t metadata() const { return _metadata; } |
461 | 4 | void set_metadata(int64_t md) { |
462 | 4 | _is_set_metadata = true; |
463 | 4 | _metadata = md; |
464 | 4 | } |
465 | | |
466 | 110 | bool is_set_metadata() const { return _is_set_metadata; } |
467 | | |
468 | 0 | time_t timestamp() const { return _timestamp; } |
469 | 0 | void set_timestamp(time_t ss) { _timestamp = ss; } |
470 | | |
471 | | // Derived counter function: return measured throughput as input_value/second. |
472 | | static int64_t units_per_second(const Counter* total_counter, const Counter* timer); |
473 | | |
474 | | // Derived counter function: return aggregated value |
475 | | static int64_t counter_sum(const std::vector<Counter*>* counters); |
476 | | |
477 | | // Function that returns a counter metric. |
478 | | // Note: this function should not block (or take a long time). |
479 | | using SampleFn = std::function<int64_t()>; |
480 | | |
481 | | // Add a rate counter to the current profile based on src_counter with name. |
482 | | // The rate counter is updated periodically based on the src counter. |
483 | | // The rate counter has units in src_counter unit per second. |
484 | | Counter* add_rate_counter(const std::string& name, Counter* src_counter); |
485 | | |
486 | | // Same as 'add_rate_counter' above except values are taken by calling fn. |
487 | | // The resulting counter will be of 'type'. |
488 | | Counter* add_rate_counter(const std::string& name, SampleFn fn, TUnit::type type); |
489 | | |
490 | | // Add a sampling counter to the current profile based on src_counter with name. |
491 | | // The sampling counter is updated periodically based on the src counter by averaging |
492 | | // the samples taken from the src counter. |
493 | | // The sampling counter has the same unit as src_counter unit. |
494 | | Counter* add_sampling_counter(const std::string& name, Counter* src_counter); |
495 | | |
496 | | // Same as 'add_sampling_counter' above except the samples are taken by calling fn. |
497 | | Counter* add_sampling_counter(const std::string& name, SampleFn fn); |
498 | | |
499 | | /// Adds a high water mark counter to the runtime profile. Otherwise, same behavior |
500 | | /// as AddCounter(). |
501 | | HighWaterMarkCounter* AddHighWaterMarkCounter(const std::string& name, TUnit::type unit, |
502 | | const std::string& parent_counter_name = "", |
503 | | int64_t level = 2); |
504 | | |
505 | | // Only for create MemTracker(using profile's counter to calc consumption) |
506 | | std::shared_ptr<HighWaterMarkCounter> AddSharedHighWaterMarkCounter( |
507 | | const std::string& name, TUnit::type unit, const std::string& parent_counter_name = ""); |
508 | | |
509 | | // Recursively compute the fraction of the 'total_time' spent in this profile and |
510 | | // its children. |
511 | | // This function updates _local_time_percent for each profile. |
512 | | void compute_time_in_profile(); |
513 | | |
514 | | void clear_children(); |
515 | | |
516 | | private: |
517 | | // Pool for allocated counters. Usually owned by the creator of this |
518 | | // object, but occasionally allocated in the constructor. |
519 | | std::unique_ptr<ObjectPool> _pool; |
520 | | |
521 | | // Pool for allocated counters. These counters are shared with some other objects. |
522 | | std::map<std::string, std::shared_ptr<HighWaterMarkCounter>> _shared_counter_pool; |
523 | | |
524 | | // Name for this runtime profile. |
525 | | std::string _name; |
526 | | |
527 | | // user-supplied, uninterpreted metadata. |
528 | | int64_t _metadata; |
529 | | bool _is_set_metadata = false; |
530 | | |
531 | | // The timestamp when the profile was modified, make sure the update is up to date. |
532 | | time_t _timestamp; |
533 | | |
534 | | /// True if this profile is an average derived from other profiles. |
535 | | /// All counters in this profile must be of unit AveragedCounter. |
536 | | bool _is_averaged_profile; |
537 | | |
538 | | // Map from counter names to counters. The profile owns the memory for the |
539 | | // counters. |
540 | | using CounterMap = std::map<std::string, Counter*>; |
541 | | CounterMap _counter_map; |
542 | | |
543 | | // Map from parent counter name to a set of child counter name. |
544 | | // All top level counters are the child of "" (root). |
545 | | using ChildCounterMap = std::map<std::string, std::set<std::string>>; |
546 | | ChildCounterMap _child_counter_map; |
547 | | |
548 | | // A set of bucket counters registered in this runtime profile. |
549 | | std::set<std::vector<Counter*>*> _bucketing_counters; |
550 | | |
551 | | // protects _counter_map, _counter_child_map and _bucketing_counters |
552 | | mutable std::mutex _counter_map_lock; |
553 | | |
554 | | // Child profiles. Does not own memory. |
555 | | // We record children in both a map (to facilitate updates) and a vector |
556 | | // (to print things in the order they were registered) |
557 | | using ChildMap = std::map<std::string, RuntimeProfile*>; |
558 | | ChildMap _child_map; |
559 | | // vector of (profile, indentation flag) |
560 | | using ChildVector = std::vector<std::pair<RuntimeProfile*, bool>>; |
561 | | ChildVector _children; |
562 | | mutable std::mutex _children_lock; // protects _child_map and _children |
563 | | |
564 | | using InfoStrings = std::map<std::string, std::string>; |
565 | | InfoStrings _info_strings; |
566 | | |
567 | | // Keeps track of the order in which InfoStrings are displayed when printed |
568 | | using InfoStringsDisplayOrder = std::vector<std::string>; |
569 | | InfoStringsDisplayOrder _info_strings_display_order; |
570 | | |
571 | | // Protects _info_strings and _info_strings_display_order |
572 | | mutable std::mutex _info_strings_lock; |
573 | | |
574 | | using EventSequenceMap = std::map<std::string, EventSequence*>; |
575 | | EventSequenceMap _event_sequence_map; |
576 | | mutable std::mutex _event_sequences_lock; |
577 | | |
578 | | Counter _counter_total_time; |
579 | | // Time spent in just in this profile (i.e. not the children) as a fraction |
580 | | // of the total time in the entire profile tree. |
581 | | double _local_time_percent; |
582 | | |
583 | | enum PeriodicCounterType { |
584 | | RATE_COUNTER = 0, |
585 | | SAMPLING_COUNTER, |
586 | | }; |
587 | | |
588 | | struct RateCounterInfo { |
589 | | Counter* src_counter = nullptr; |
590 | | SampleFn sample_fn; |
591 | | int64_t elapsed_ms; |
592 | | }; |
593 | | |
594 | | struct SamplingCounterInfo { |
595 | | Counter* src_counter = nullptr; // the counter to be sampled |
596 | | SampleFn sample_fn; |
597 | | int64_t total_sampled_value; // sum of all sampled values; |
598 | | int64_t num_sampled; // number of samples taken |
599 | | }; |
600 | | |
601 | | struct BucketCountersInfo { |
602 | | Counter* src_counter = nullptr; // the counter to be sampled |
603 | | int64_t num_sampled; // number of samples taken |
604 | | // TODO: customize bucketing |
605 | | }; |
606 | | |
607 | | // update a subtree of profiles from nodes, rooted at *idx. |
608 | | // On return, *idx points to the node immediately following this subtree. |
609 | | void update(const std::vector<TRuntimeProfileNode>& nodes, int* idx); |
610 | | |
611 | | // Helper function to compute compute the fraction of the total time spent in |
612 | | // this profile and its children. |
613 | | // Called recursively. |
614 | | void compute_time_in_profile(int64_t total_time); |
615 | | |
616 | | // Print the child counters of the given counter name |
617 | | static void print_child_counters(const std::string& prefix, const std::string& counter_name, |
618 | | const CounterMap& counter_map, |
619 | | const ChildCounterMap& child_counter_map, std::ostream* s); |
620 | | }; |
621 | | |
622 | | // Utility class to update the counter at object construction and destruction. |
623 | | // When the object is constructed, decrement the counter by val. |
624 | | // When the object goes out of scope, increment the counter by val. |
625 | | class ScopedCounter { |
626 | | public: |
627 | 0 | ScopedCounter(RuntimeProfile::Counter* counter, int64_t val) : _val(val), _counter(counter) { |
628 | 0 | if (counter == nullptr) { |
629 | 0 | return; |
630 | 0 | } |
631 | 0 |
|
632 | 0 | _counter->update(-1L * _val); |
633 | 0 | } |
634 | | |
635 | | // Increment the counter when object is destroyed |
636 | 0 | ~ScopedCounter() { |
637 | 0 | if (_counter != nullptr) { |
638 | 0 | _counter->update(_val); |
639 | 0 | } |
640 | 0 | } |
641 | | |
642 | | // Disable copy constructor and assignment |
643 | | ScopedCounter(const ScopedCounter& counter) = delete; |
644 | | ScopedCounter& operator=(const ScopedCounter& counter) = delete; |
645 | | |
646 | | private: |
647 | | int64_t _val; |
648 | | RuntimeProfile::Counter* _counter = nullptr; |
649 | | }; |
650 | | |
651 | | // Utility class to update time elapsed when the object goes out of scope. |
652 | | // 'T' must implement the stopWatch "interface" (start,stop,elapsed_time) but |
653 | | // we use templates not to pay for virtual function overhead. |
654 | | template <class T, typename Bool = bool> |
655 | | class ScopedTimer { |
656 | | public: |
657 | | ScopedTimer(RuntimeProfile::Counter* counter, const Bool* is_cancelled = nullptr) |
658 | 966 | : _counter(counter), _is_cancelled(is_cancelled) { |
659 | 966 | if (counter == nullptr) { |
660 | 41 | return; |
661 | 41 | } |
662 | 925 | DCHECK_EQ(counter->type(), TUnit::TIME_NS); |
663 | 925 | _sw.start(); |
664 | 925 | } _ZN5doris11ScopedTimerINS_15CustomStopWatchILi1EEEbEC2EPNS_14RuntimeProfile7CounterEPKb Line | Count | Source | 658 | 966 | : _counter(counter), _is_cancelled(is_cancelled) { | 659 | 966 | if (counter == nullptr) { | 660 | 41 | return; | 661 | 41 | } | 662 | 925 | DCHECK_EQ(counter->type(), TUnit::TIME_NS); | 663 | 925 | _sw.start(); | 664 | 925 | } |
Unexecuted instantiation: _ZN5doris11ScopedTimerINS_15CustomStopWatchILi3EEEbEC2EPNS_14RuntimeProfile7CounterEPKb |
665 | | |
666 | | void stop() { _sw.stop(); } |
667 | | |
668 | | void start() { _sw.start(); } |
669 | | |
670 | 925 | bool is_cancelled() { return _is_cancelled != nullptr && *_is_cancelled; } _ZN5doris11ScopedTimerINS_15CustomStopWatchILi1EEEbE12is_cancelledEv Line | Count | Source | 670 | 925 | bool is_cancelled() { return _is_cancelled != nullptr && *_is_cancelled; } |
Unexecuted instantiation: _ZN5doris11ScopedTimerINS_15CustomStopWatchILi3EEEbE12is_cancelledEv |
671 | | |
672 | 925 | void UpdateCounter() { |
673 | 925 | if (_counter != nullptr && !is_cancelled()) { |
674 | 925 | _counter->update(_sw.elapsed_time()); |
675 | 925 | } |
676 | 925 | } _ZN5doris11ScopedTimerINS_15CustomStopWatchILi1EEEbE13UpdateCounterEv Line | Count | Source | 672 | 925 | void UpdateCounter() { | 673 | 925 | if (_counter != nullptr && !is_cancelled()) { | 674 | 925 | _counter->update(_sw.elapsed_time()); | 675 | 925 | } | 676 | 925 | } |
Unexecuted instantiation: _ZN5doris11ScopedTimerINS_15CustomStopWatchILi3EEEbE13UpdateCounterEv |
677 | | |
678 | | // Update counter when object is destroyed |
679 | 966 | ~ScopedTimer() { |
680 | 966 | if (_counter == nullptr) { |
681 | 41 | return; |
682 | 41 | } |
683 | 925 | _sw.stop(); |
684 | 925 | UpdateCounter(); |
685 | 925 | } _ZN5doris11ScopedTimerINS_15CustomStopWatchILi1EEEbED2Ev Line | Count | Source | 679 | 966 | ~ScopedTimer() { | 680 | 966 | if (_counter == nullptr) { | 681 | 41 | return; | 682 | 41 | } | 683 | 925 | _sw.stop(); | 684 | 925 | UpdateCounter(); | 685 | 925 | } |
Unexecuted instantiation: _ZN5doris11ScopedTimerINS_15CustomStopWatchILi3EEEbED2Ev |
686 | | |
687 | | // Disable copy constructor and assignment |
688 | | ScopedTimer(const ScopedTimer& timer) = delete; |
689 | | ScopedTimer& operator=(const ScopedTimer& timer) = delete; |
690 | | |
691 | | private: |
692 | | T _sw; |
693 | | RuntimeProfile::Counter* _counter = nullptr; |
694 | | const Bool* _is_cancelled = nullptr; |
695 | | }; |
696 | | |
697 | | // Utility class to update time elapsed when the object goes out of scope. |
698 | | // 'T' must implement the stopWatch "interface" (start,stop,elapsed_time) but |
699 | | // we use templates not to pay for virtual function overhead. |
700 | | template <class T, class C> |
701 | | class ScopedRawTimer { |
702 | | public: |
703 | 259k | ScopedRawTimer(C* counter) : _counter(counter) { _sw.start(); } _ZN5doris14ScopedRawTimerINS_15CustomStopWatchILi1EEElEC2EPl Line | Count | Source | 703 | 259k | ScopedRawTimer(C* counter) : _counter(counter) { _sw.start(); } |
Unexecuted instantiation: _ZN5doris14ScopedRawTimerINS_15CustomStopWatchILi1EEESt6atomicIlEEC2EPS4_ |
704 | | // Update counter when object is destroyed |
705 | 259k | ~ScopedRawTimer() { *_counter += _sw.elapsed_time(); } _ZN5doris14ScopedRawTimerINS_15CustomStopWatchILi1EEElED2Ev Line | Count | Source | 705 | 259k | ~ScopedRawTimer() { *_counter += _sw.elapsed_time(); } |
Unexecuted instantiation: _ZN5doris14ScopedRawTimerINS_15CustomStopWatchILi1EEESt6atomicIlEED2Ev |
706 | | |
707 | | // Disable copy constructor and assignment |
708 | | ScopedRawTimer(const ScopedRawTimer& timer) = delete; |
709 | | ScopedRawTimer& operator=(const ScopedRawTimer& timer) = delete; |
710 | | |
711 | | private: |
712 | | T _sw; |
713 | | C* _counter = nullptr; |
714 | | }; |
715 | | |
716 | | } // namespace doris |