/root/doris/be/src/pipeline/pipeline_tracing.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <concurrentqueue.h> |
21 | | #include <fmt/format.h> |
22 | | #include <gen_cpp/Types_types.h> |
23 | | #include <parallel_hashmap/phmap.h> |
24 | | |
25 | | #include <cstdint> |
26 | | #include <filesystem> |
27 | | |
28 | | #include "common/config.h" |
29 | | #include "util/hash_util.hpp" // IWYU pragma: keep |
30 | | #include "util/thrift_util.h" |
31 | | #include "util/time.h" |
32 | | |
33 | | namespace doris::pipeline { |
34 | | |
35 | | struct ScheduleRecord { |
36 | | TUniqueId query_id; |
37 | | std::string task_id; |
38 | | uint32_t core_id; |
39 | | uint64_t thread_id; |
40 | | uint64_t start_time; |
41 | | uint64_t end_time; |
42 | | |
43 | 0 | bool operator<(const ScheduleRecord& rhs) const { return start_time < rhs.start_time; } |
44 | 0 | std::string to_string(uint64_t append_value) const { |
45 | 0 | return fmt::format("{}|{}|{}|{}|{}|{}|{}\n", doris::to_string(query_id), task_id, core_id, |
46 | 0 | thread_id, start_time, end_time, append_value); |
47 | 0 | } |
48 | | }; |
49 | | |
50 | | struct QueryID { |
51 | | TUniqueId query_id; |
52 | 0 | bool operator<(const QueryID& query_id_) const { |
53 | 0 | return query_id.hi < query_id_.query_id.hi || |
54 | 0 | (query_id.hi == query_id_.query_id.hi && query_id.lo < query_id_.query_id.lo); |
55 | 0 | } |
56 | 0 | bool operator==(const QueryID& query_id_) const { return query_id == query_id_.query_id; } |
57 | | }; |
58 | | |
59 | | // all tracing datas of ONE specific query |
60 | | using OneQueryTraces = moodycamel::ConcurrentQueue<ScheduleRecord>; |
61 | | using OneQueryTracesSPtr = std::shared_ptr<moodycamel::ConcurrentQueue<ScheduleRecord>>; |
62 | | using QueryTracesMap = std::map<QueryID, OneQueryTracesSPtr>; |
63 | | |
64 | | // belongs to exec_env, for all query, if enabled |
65 | | class PipelineTracerContext { |
66 | | public: |
67 | 0 | PipelineTracerContext() : _data(std::make_shared<QueryTracesMap>()) {} |
68 | | enum class RecordType { |
69 | | None, // disable |
70 | | PerQuery, // record per query. one query one file. |
71 | | Periodic // record per times. one timeslice one file. |
72 | | }; |
73 | | void record(ScheduleRecord record); // record one schedule record |
74 | | void end_query(TUniqueId query_id, |
75 | | uint64_t workload_group); // tell context this query is end. may leads to dump. |
76 | | Status change_record_params(const std::map<std::string, std::string>& params); |
77 | | |
78 | 0 | bool enabled() const { return !(_dump_type == RecordType::None); } |
79 | | |
80 | | private: |
81 | | // dump data to disk. one query or all. |
82 | | void _dump_query(TUniqueId query_id); |
83 | | void _dump_timeslice(); |
84 | | void _update(std::function<void(QueryTracesMap&)>&& handler); |
85 | | |
86 | | std::filesystem::path _log_dir = fmt::format("{}/pipe_tracing", getenv("LOG_DIR")); |
87 | | |
88 | | std::shared_ptr<QueryTracesMap> _data; |
89 | | std::mutex _tg_lock; //TODO: use an lockfree DS |
90 | | phmap::flat_hash_map<TUniqueId, uint64_t> |
91 | | _id_to_workload_group; // save query's workload group number |
92 | | |
93 | | RecordType _dump_type = RecordType::None; |
94 | | decltype(MonotonicSeconds()) _last_dump_time; |
95 | | decltype(MonotonicSeconds()) _dump_interval_s = |
96 | | 60; // effective iff Periodic mode. 1 minute default. |
97 | | }; |
98 | | } // namespace doris::pipeline |