/root/doris/be/src/runtime/exec_env.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <common/multi_version.h> |
21 | | |
22 | | #include <atomic> |
23 | | #include <map> |
24 | | #include <memory> |
25 | | #include <mutex> |
26 | | #include <string> |
27 | | #include <vector> |
28 | | |
29 | | #include "common/status.h" |
30 | | #include "io/cache/fs_file_cache_storage.h" |
31 | | #include "olap/memtable_memory_limiter.h" |
32 | | #include "olap/options.h" |
33 | | #include "olap/rowset/segment_v2/inverted_index_writer.h" |
34 | | #include "olap/tablet_fwd.h" |
35 | | #include "pipeline/pipeline_tracing.h" |
36 | | #include "runtime/cluster_info.h" |
37 | | #include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header |
38 | | #include "util/threadpool.h" |
39 | | |
40 | | namespace orc { |
41 | | class MemoryPool; |
42 | | } |
43 | | namespace arrow { |
44 | | class MemoryPool; |
45 | | } |
46 | | |
47 | | namespace doris { |
48 | | namespace vectorized { |
49 | | class VDataStreamMgr; |
50 | | class ScannerScheduler; |
51 | | class SpillStreamManager; |
52 | | class DeltaWriterV2Pool; |
53 | | } // namespace vectorized |
54 | | namespace pipeline { |
55 | | class TaskScheduler; |
56 | | struct RuntimeFilterTimerQueue; |
57 | | } // namespace pipeline |
58 | | class WorkloadGroupMgr; |
59 | | struct WriteCooldownMetaExecutors; |
60 | | namespace io { |
61 | | class FileCacheFactory; |
62 | | class HdfsMgr; |
63 | | } // namespace io |
64 | | namespace segment_v2 { |
65 | | class InvertedIndexSearcherCache; |
66 | | class InvertedIndexQueryCache; |
67 | | class TmpFileDirs; |
68 | | } // namespace segment_v2 |
69 | | |
70 | | namespace kerberos { |
71 | | class KerberosTicketMgr; |
72 | | } |
73 | | |
74 | | class QueryCache; |
75 | | class WorkloadSchedPolicyMgr; |
76 | | class BfdParser; |
77 | | class BrokerMgr; |
78 | | template <class T> |
79 | | class BrpcClientCache; |
80 | | class ExternalScanContextMgr; |
81 | | class FragmentMgr; |
82 | | class ResultCache; |
83 | | class LoadPathMgr; |
84 | | class NewLoadStreamMgr; |
85 | | class MemTrackerLimiter; |
86 | | class MemTracker; |
87 | | struct TrackerLimiterGroup; |
88 | | class BaseStorageEngine; |
89 | | class ResultBufferMgr; |
90 | | class ResultQueueMgr; |
91 | | class RuntimeQueryStatisticsMgr; |
92 | | class LoadChannelMgr; |
93 | | class LoadStreamMgr; |
94 | | class LoadStreamMapPool; |
95 | | class StreamLoadExecutor; |
96 | | class RoutineLoadTaskExecutor; |
97 | | class SmallFileMgr; |
98 | | class BackendServiceClient; |
99 | | class TPaloBrokerServiceClient; |
100 | | class PBackendService_Stub; |
101 | | class PFunctionService_Stub; |
102 | | template <class T> |
103 | | class ClientCache; |
104 | | class HeartbeatFlags; |
105 | | class FrontendServiceClient; |
106 | | class FileMetaCache; |
107 | | class GroupCommitMgr; |
108 | | class TabletSchemaCache; |
109 | | class TabletColumnObjectPool; |
110 | | class UserFunctionCache; |
111 | | class SchemaCache; |
112 | | class StoragePageCache; |
113 | | class SegmentLoader; |
114 | | class LookupConnectionCache; |
115 | | class RowCache; |
116 | | class DummyLRUCache; |
117 | | class CacheManager; |
118 | | class ProcessProfile; |
119 | | class HeapProfiler; |
120 | | class WalManager; |
121 | | class DNSCache; |
122 | | |
123 | | inline bool k_doris_exit = false; |
124 | | |
125 | | // Execution environment for queries/plan fragments. |
126 | | // Contains all required global structures, and handles to |
127 | | // singleton services. Clients must call StartServices exactly |
128 | | // once to properly initialise service state. |
129 | | class ExecEnv { |
130 | | public: |
131 | | // Empty destructor because the compiler-generated one requires full |
132 | | // declarations for classes in scoped_ptrs. |
133 | | ~ExecEnv(); |
134 | | |
135 | 8.85k | BaseStorageEngine& storage_engine() { return *_storage_engine; } |
136 | | |
137 | | // Initial exec environment. must call this to init all |
138 | | [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths, |
139 | | const std::vector<StorePath>& spill_store_paths, |
140 | | const std::set<std::string>& broken_paths); |
141 | | |
142 | | // Stop all threads and delete resources. |
143 | | void destroy(); |
144 | | |
145 | | /// Returns the first created exec env instance. In a normal doris, this is |
146 | | /// the only instance. In test setups with multiple ExecEnv's per process, |
147 | | /// we return the most recently created instance. |
148 | 85.6k | static ExecEnv* GetInstance() { |
149 | 85.6k | static ExecEnv s_exec_env; |
150 | 85.6k | return &s_exec_env; |
151 | 85.6k | } |
152 | | |
153 | | // Requires ExenEnv ready |
154 | | static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id); |
155 | | |
156 | 2.42M | static bool ready() { return _s_ready.load(std::memory_order_acquire); } |
157 | 12 | static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); } |
158 | | const std::string& token() const; |
159 | 0 | ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; } |
160 | 8 | vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; } |
161 | 0 | ResultBufferMgr* result_mgr() { return _result_mgr; } |
162 | 1 | ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; } |
163 | 0 | ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; } |
164 | 0 | ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; } |
165 | 0 | ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; } |
166 | | |
167 | 0 | pipeline::TaskScheduler* pipeline_task_scheduler() { return _without_group_task_scheduler; } |
168 | 0 | WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; } |
169 | 0 | WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; } |
170 | 0 | RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() { |
171 | 0 | return _runtime_query_statistics_mgr; |
172 | 0 | } |
173 | | |
174 | | // using template to simplify client cache management |
175 | | template <typename T> |
176 | | inline ClientCache<T>* get_client_cache() { |
177 | | return nullptr; |
178 | | } |
179 | | |
180 | | // Save all MemTrackerLimiters in use. |
181 | | // Each group corresponds to several MemTrackerLimiters and has a lock. |
182 | | // Multiple groups are used to reduce the impact of locks. |
183 | | std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool; |
184 | | void init_mem_tracker(); |
185 | 101 | std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; } |
186 | 0 | std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() { |
187 | 0 | return _brpc_iobuf_block_memory_tracker; |
188 | 0 | } |
189 | 0 | std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() { |
190 | 0 | return _segcompaction_mem_tracker; |
191 | 0 | } |
192 | 0 | std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() { |
193 | 0 | return _stream_load_pipe_tracker; |
194 | 0 | } |
195 | 0 | std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() { |
196 | 0 | return _tablets_no_cache_mem_tracker; |
197 | 0 | } |
198 | 0 | std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() { |
199 | 0 | return _rowsets_no_cache_mem_tracker; |
200 | 0 | } |
201 | 0 | std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() { |
202 | 0 | return _segments_no_cache_mem_tracker; |
203 | 0 | } |
204 | 0 | std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() { |
205 | 0 | return _point_query_executor_mem_tracker; |
206 | 0 | } |
207 | 0 | std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() { |
208 | 0 | return _query_cache_mem_tracker; |
209 | 0 | } |
210 | 0 | std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() { |
211 | 0 | return _block_compression_mem_tracker; |
212 | 0 | } |
213 | 0 | std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() { |
214 | 0 | return _rowid_storage_reader_tracker; |
215 | 0 | } |
216 | 0 | std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() { |
217 | 0 | return _subcolumns_tree_tracker; |
218 | 0 | } |
219 | 0 | std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; } |
220 | | |
221 | 0 | ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); } |
222 | 4 | ThreadPool* buffered_reader_prefetch_thread_pool() { |
223 | 4 | return _buffered_reader_prefetch_thread_pool.get(); |
224 | 4 | } |
225 | 0 | ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); } |
226 | 279 | ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); } |
227 | 0 | ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); } |
228 | | ThreadPool* non_block_close_thread_pool(); |
229 | 0 | ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); } |
230 | | |
231 | | Status init_pipeline_task_scheduler(); |
232 | | void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths); |
233 | 72 | io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; } |
234 | 0 | UserFunctionCache* user_function_cache() { return _user_function_cache; } |
235 | 8 | FragmentMgr* fragment_mgr() { return _fragment_mgr; } |
236 | 0 | ResultCache* result_cache() { return _result_cache; } |
237 | 12 | ClusterInfo* cluster_info() { return _cluster_info; } |
238 | 1 | LoadPathMgr* load_path_mgr() { return _load_path_mgr; } |
239 | 1 | BfdParser* bfd_parser() const { return _bfd_parser; } |
240 | 0 | BrokerMgr* broker_mgr() const { return _broker_mgr; } |
241 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const { |
242 | 0 | return _internal_client_cache; |
243 | 0 | } |
244 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const { |
245 | 0 | return _streaming_client_cache; |
246 | 0 | } |
247 | 0 | BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const { |
248 | 0 | return _function_client_cache; |
249 | 0 | } |
250 | 0 | LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; } |
251 | 0 | LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); } |
252 | 3 | NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); } |
253 | 0 | SmallFileMgr* small_file_mgr() { return _small_file_mgr; } |
254 | 223 | doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; } |
255 | 0 | GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; } |
256 | | |
257 | 0 | const std::vector<StorePath>& store_paths() const { return _store_paths; } |
258 | | |
259 | 1 | StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); } |
260 | 0 | RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; } |
261 | 0 | HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; } |
262 | 14 | vectorized::ScannerScheduler* scanner_scheduler() { return _scanner_scheduler; } |
263 | 0 | FileMetaCache* file_meta_cache() { return _file_meta_cache; } |
264 | 316 | MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); } |
265 | 47 | WalManager* wal_mgr() { return _wal_manager.get(); } |
266 | 8 | DNSCache* dns_cache() { return _dns_cache; } |
267 | 5 | WriteCooldownMetaExecutors* write_cooldown_meta_executors() { |
268 | 5 | return _write_cooldown_meta_executors.get(); |
269 | 5 | } |
270 | | |
271 | 0 | kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } |
272 | 0 | io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } |
273 | | |
274 | | #ifdef BE_TEST |
275 | 81 | void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) { |
276 | 81 | this->_tmp_file_dirs = std::move(tmp_file_dirs); |
277 | 81 | } |
278 | 4 | void set_ready() { this->_s_ready = true; } |
279 | 0 | void set_not_ready() { this->_s_ready = false; } |
280 | 34 | void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { |
281 | 34 | _memtable_memory_limiter.reset(limiter); |
282 | 34 | } |
283 | 4 | void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } |
284 | | void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr); |
285 | | void clear_new_load_stream_mgr(); |
286 | | void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor); |
287 | | void clear_stream_load_executor(); |
288 | | |
289 | | void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine); |
290 | | void set_inverted_index_searcher_cache( |
291 | | segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); |
292 | 1 | void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } |
293 | 1 | void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } |
294 | 1 | void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } |
295 | 1 | void set_tablet_column_object_pool(TabletColumnObjectPool* c) { |
296 | 1 | this->_tablet_column_object_pool = c; |
297 | 1 | } |
298 | 1 | void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } |
299 | 1 | void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } |
300 | 0 | void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { |
301 | 0 | this->_routine_load_task_executor = r; |
302 | 0 | } |
303 | | void set_wal_mgr(std::unique_ptr<WalManager>&& wm); |
304 | | void clear_wal_mgr(); |
305 | | |
306 | | void set_write_cooldown_meta_executors(); |
307 | 1 | static void set_tracking_memory(bool tracking_memory) { |
308 | 1 | _s_tracking_memory.store(tracking_memory, std::memory_order_release); |
309 | 1 | } |
310 | 1 | void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } |
311 | 2 | void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
312 | 2 | _non_block_close_thread_pool = std::move(pool); |
313 | 2 | } |
314 | | #endif |
315 | 0 | LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); } |
316 | | |
317 | 0 | vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); } |
318 | | |
319 | | void wait_for_all_tasks_done(); |
320 | | |
321 | | void update_frontends(const std::vector<TFrontendInfo>& new_infos); |
322 | | std::vector<TFrontendInfo> get_frontends(); |
323 | | std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); |
324 | | |
325 | 6.64k | TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } |
326 | 359 | TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } |
327 | 0 | SchemaCache* schema_cache() { return _schema_cache; } |
328 | 26.7k | StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } |
329 | 27.6k | SegmentLoader* segment_loader() { return _segment_loader; } |
330 | 0 | LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; } |
331 | 0 | RowCache* get_row_cache() { return _row_cache; } |
332 | 959 | CacheManager* get_cache_manager() { return _cache_manager; } |
333 | 2 | ProcessProfile* get_process_profile() { return _process_profile; } |
334 | 0 | HeapProfiler* get_heap_profiler() { return _heap_profiler; } |
335 | 182 | segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() { |
336 | 182 | return _inverted_index_searcher_cache; |
337 | 182 | } |
338 | 44 | segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() { |
339 | 44 | return _inverted_index_query_cache; |
340 | 44 | } |
341 | 0 | QueryCache* get_query_cache() { return _query_cache; } |
342 | | |
343 | 0 | pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { |
344 | 0 | return _runtime_filter_timer_queue; |
345 | 0 | } |
346 | | |
347 | 0 | pipeline::PipelineTracerContext* pipeline_tracer_context() { |
348 | 0 | return _pipeline_tracer_ctx.get(); |
349 | 0 | } |
350 | | |
351 | 316 | segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); } |
352 | 3.76k | io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); } |
353 | | |
354 | 13 | orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; } |
355 | 0 | arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; } |
356 | | |
357 | | bool check_auth_token(const std::string& auth_token); |
358 | 3 | void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; } |
359 | | void clear_stream_mgr(); |
360 | | |
361 | | private: |
362 | | ExecEnv(); |
363 | | |
364 | | [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths, |
365 | | const std::vector<StorePath>& spill_store_paths, |
366 | | const std::set<std::string>& broken_paths); |
367 | | void _destroy(); |
368 | | |
369 | | Status _init_mem_env(); |
370 | | Status _check_deploy_mode(); |
371 | | |
372 | | void _register_metrics(); |
373 | | void _deregister_metrics(); |
374 | | |
375 | | inline static std::atomic_bool _s_ready {false}; |
376 | | inline static std::atomic_bool _s_tracking_memory {false}; |
377 | | std::vector<StorePath> _store_paths; |
378 | | std::vector<StorePath> _spill_store_paths; |
379 | | |
380 | | io::FileCacheFactory* _file_cache_factory = nullptr; |
381 | | UserFunctionCache* _user_function_cache = nullptr; |
382 | | // Leave protected so that subclasses can override |
383 | | ExternalScanContextMgr* _external_scan_context_mgr = nullptr; |
384 | | vectorized::VDataStreamMgr* _vstream_mgr = nullptr; |
385 | | ResultBufferMgr* _result_mgr = nullptr; |
386 | | ResultQueueMgr* _result_queue_mgr = nullptr; |
387 | | ClientCache<BackendServiceClient>* _backend_client_cache = nullptr; |
388 | | ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr; |
389 | | ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr; |
390 | | |
391 | | // The default tracker consumed by mem hook. If the thread does not attach other trackers, |
392 | | // by default all consumption will be passed to the process tracker through the orphan tracker. |
393 | | // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`. |
394 | | // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership", |
395 | | // and the consumption of the orphan mem tracker is close to 0, but greater than 0. |
396 | | std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker; |
397 | | std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker; |
398 | | // Count the memory consumption of segment compaction tasks. |
399 | | std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker; |
400 | | std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker; |
401 | | |
402 | | std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker; |
403 | | std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker; |
404 | | std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker; |
405 | | |
406 | | // Tracking memory may be shared between multiple queries. |
407 | | std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker; |
408 | | std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker; |
409 | | std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker; |
410 | | |
411 | | // TODO, looking forward to more accurate tracking. |
412 | | std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker; |
413 | | std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker; |
414 | | std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker; |
415 | | |
416 | | std::unique_ptr<ThreadPool> _send_batch_thread_pool; |
417 | | // Threadpool used to prefetch remote file for buffered reader |
418 | | std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool; |
419 | | // Threadpool used to send TableStats to FE |
420 | | std::unique_ptr<ThreadPool> _send_table_stats_thread_pool; |
421 | | // Threadpool used to upload local file to s3 |
422 | | std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool; |
423 | | // Pool used by join node to build hash table |
424 | | // Pool to use a new thread to release object |
425 | | std::unique_ptr<ThreadPool> _lazy_release_obj_pool; |
426 | | std::unique_ptr<ThreadPool> _non_block_close_thread_pool; |
427 | | std::unique_ptr<ThreadPool> _s3_file_system_thread_pool; |
428 | | |
429 | | FragmentMgr* _fragment_mgr = nullptr; |
430 | | pipeline::TaskScheduler* _without_group_task_scheduler = nullptr; |
431 | | WorkloadGroupMgr* _workload_group_manager = nullptr; |
432 | | |
433 | | ResultCache* _result_cache = nullptr; |
434 | | ClusterInfo* _cluster_info = nullptr; |
435 | | LoadPathMgr* _load_path_mgr = nullptr; |
436 | | |
437 | | BfdParser* _bfd_parser = nullptr; |
438 | | BrokerMgr* _broker_mgr = nullptr; |
439 | | LoadChannelMgr* _load_channel_mgr = nullptr; |
440 | | std::unique_ptr<LoadStreamMgr> _load_stream_mgr; |
441 | | std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr; |
442 | | BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr; |
443 | | BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr; |
444 | | BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr; |
445 | | |
446 | | std::unique_ptr<StreamLoadExecutor> _stream_load_executor; |
447 | | RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr; |
448 | | SmallFileMgr* _small_file_mgr = nullptr; |
449 | | HeartbeatFlags* _heartbeat_flags = nullptr; |
450 | | vectorized::ScannerScheduler* _scanner_scheduler = nullptr; |
451 | | |
452 | | // To save meta info of external file, such as parquet footer. |
453 | | FileMetaCache* _file_meta_cache = nullptr; |
454 | | std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter; |
455 | | std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool; |
456 | | std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool; |
457 | | std::unique_ptr<WalManager> _wal_manager; |
458 | | DNSCache* _dns_cache = nullptr; |
459 | | std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors; |
460 | | |
461 | | std::mutex _frontends_lock; |
462 | | // ip:brpc_port -> frontend_indo |
463 | | std::map<TNetworkAddress, FrontendInfo> _frontends; |
464 | | GroupCommitMgr* _group_commit_mgr = nullptr; |
465 | | |
466 | | // Maybe we should use unique_ptr, but it need complete type, which means we need |
467 | | // to include many headers, and for some cpp file that do not need class like TabletSchemaCache, |
468 | | // these redundancy header could introduce potential bug, at least, more header means slow compile. |
469 | | // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. |
470 | | TabletSchemaCache* _tablet_schema_cache = nullptr; |
471 | | TabletColumnObjectPool* _tablet_column_object_pool = nullptr; |
472 | | std::unique_ptr<BaseStorageEngine> _storage_engine; |
473 | | SchemaCache* _schema_cache = nullptr; |
474 | | StoragePageCache* _storage_page_cache = nullptr; |
475 | | SegmentLoader* _segment_loader = nullptr; |
476 | | LookupConnectionCache* _lookup_connection_cache = nullptr; |
477 | | RowCache* _row_cache = nullptr; |
478 | | CacheManager* _cache_manager = nullptr; |
479 | | ProcessProfile* _process_profile = nullptr; |
480 | | HeapProfiler* _heap_profiler = nullptr; |
481 | | segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr; |
482 | | segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr; |
483 | | QueryCache* _query_cache = nullptr; |
484 | | std::unique_ptr<io::FDCache> _file_cache_open_fd_cache; |
485 | | |
486 | | pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; |
487 | | |
488 | | WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; |
489 | | |
490 | | RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr; |
491 | | |
492 | | std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx; |
493 | | std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs; |
494 | | doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr; |
495 | | |
496 | | orc::MemoryPool* _orc_memory_pool = nullptr; |
497 | | arrow::MemoryPool* _arrow_memory_pool = nullptr; |
498 | | |
499 | | kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; |
500 | | io::HdfsMgr* _hdfs_mgr = nullptr; |
501 | | }; |
502 | | |
503 | | template <> |
504 | 0 | inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() { |
505 | 0 | return _backend_client_cache; |
506 | 0 | } |
507 | | template <> |
508 | 0 | inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() { |
509 | 0 | return _frontend_client_cache; |
510 | 0 | } |
511 | | template <> |
512 | | inline ClientCache<TPaloBrokerServiceClient>* |
513 | 0 | ExecEnv::get_client_cache<TPaloBrokerServiceClient>() { |
514 | 0 | return _broker_client_cache; |
515 | 0 | } |
516 | | |
517 | 0 | inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() { |
518 | 0 | return ExecEnv::GetInstance()->get_inverted_index_query_cache(); |
519 | 0 | } |
520 | | |
521 | | } // namespace doris |