/root/doris/be/src/runtime/exec_env.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <common/multi_version.h> |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <map> |
25 | | #include <memory> |
26 | | #include <mutex> |
27 | | #include <optional> |
28 | | #include <string> |
29 | | #include <vector> |
30 | | |
31 | | #include "common/config.h" |
32 | | #include "common/status.h" |
33 | | #include "io/cache/fs_file_cache_storage.h" |
34 | | #include "olap/memtable_memory_limiter.h" |
35 | | #include "olap/options.h" |
36 | | #include "olap/rowset/segment_v2/inverted_index_writer.h" |
37 | | #include "olap/tablet_fwd.h" |
38 | | #include "pipeline/pipeline_tracing.h" |
39 | | #include "runtime/cluster_info.h" |
40 | | #include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header |
41 | | #include "util/threadpool.h" |
42 | | |
43 | | namespace orc { |
44 | | class MemoryPool; |
45 | | } |
46 | | namespace arrow { |
47 | | class MemoryPool; |
48 | | } |
49 | | |
50 | | namespace doris { |
51 | | namespace vectorized { |
52 | | class VDataStreamMgr; |
53 | | class ScannerScheduler; |
54 | | class SpillStreamManager; |
55 | | class DeltaWriterV2Pool; |
56 | | class DictionaryFactory; |
57 | | } // namespace vectorized |
58 | | namespace pipeline { |
59 | | class TaskScheduler; |
60 | | struct RuntimeFilterTimerQueue; |
61 | | } // namespace pipeline |
62 | | class WorkloadGroupMgr; |
63 | | struct WriteCooldownMetaExecutors; |
64 | | namespace io { |
65 | | class FileCacheFactory; |
66 | | class HdfsMgr; |
67 | | } // namespace io |
68 | | namespace segment_v2 { |
69 | | class InvertedIndexSearcherCache; |
70 | | class InvertedIndexQueryCache; |
71 | | class TmpFileDirs; |
72 | | |
73 | | namespace inverted_index { |
74 | | class AnalysisFactoryMgr; |
75 | | } |
76 | | |
77 | | } // namespace segment_v2 |
78 | | |
79 | | namespace kerberos { |
80 | | class KerberosTicketMgr; |
81 | | } |
82 | | |
83 | | class QueryCache; |
84 | | class WorkloadSchedPolicyMgr; |
85 | | class BfdParser; |
86 | | class BrokerMgr; |
87 | | template <class T> |
88 | | class BrpcClientCache; |
89 | | class ExternalScanContextMgr; |
90 | | class FragmentMgr; |
91 | | class ResultCache; |
92 | | class LoadPathMgr; |
93 | | class NewLoadStreamMgr; |
94 | | class MemTrackerLimiter; |
95 | | class MemTracker; |
96 | | struct TrackerLimiterGroup; |
97 | | class BaseStorageEngine; |
98 | | class ResultBufferMgr; |
99 | | class ResultQueueMgr; |
100 | | class RuntimeQueryStatisticsMgr; |
101 | | class LoadChannelMgr; |
102 | | class LoadStreamMgr; |
103 | | class LoadStreamMapPool; |
104 | | class StreamLoadExecutor; |
105 | | class RoutineLoadTaskExecutor; |
106 | | class SmallFileMgr; |
107 | | class BackendServiceClient; |
108 | | class TPaloBrokerServiceClient; |
109 | | class PBackendService_Stub; |
110 | | class PFunctionService_Stub; |
111 | | template <class T> |
112 | | class ClientCache; |
113 | | class HeartbeatFlags; |
114 | | class FrontendServiceClient; |
115 | | class FileMetaCache; |
116 | | class GroupCommitMgr; |
117 | | class TabletSchemaCache; |
118 | | class TabletColumnObjectPool; |
119 | | class UserFunctionCache; |
120 | | class SchemaCache; |
121 | | class StoragePageCache; |
122 | | class SegmentLoader; |
123 | | class LookupConnectionCache; |
124 | | class RowCache; |
125 | | class DummyLRUCache; |
126 | | class CacheManager; |
127 | | class IdManager; |
128 | | class ProcessProfile; |
129 | | class HeapProfiler; |
130 | | class WalManager; |
131 | | class DNSCache; |
132 | | class IndexPolicyMgr; |
133 | | struct SyncRowsetStats; |
134 | | class DeleteBitmapAggCache; |
135 | | |
136 | | inline bool k_doris_exit = false; |
137 | | |
138 | | // Execution environment for queries/plan fragments. |
139 | | // Contains all required global structures, and handles to |
140 | | // singleton services. Clients must call StartServices exactly |
141 | | // once to properly initialise service state. |
142 | | class ExecEnv { |
143 | | public: |
144 | | // Empty destructor because the compiler-generated one requires full |
145 | | // declarations for classes in scoped_ptrs. |
146 | | ~ExecEnv(); |
147 | | |
148 | 8.92k | BaseStorageEngine& storage_engine() { return *_storage_engine; } |
149 | | |
150 | | // Initial exec environment. must call this to init all |
151 | | [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths, |
152 | | const std::vector<StorePath>& spill_store_paths, |
153 | | const std::set<std::string>& broken_paths); |
154 | | |
155 | | // Stop all threads and delete resources. |
156 | | void destroy(); |
157 | | |
158 | | /// Returns the first created exec env instance. In a normal doris, this is |
159 | | /// the only instance. In test setups with multiple ExecEnv's per process, |
160 | | /// we return the most recently created instance. |
161 | 563k | static ExecEnv* GetInstance() { |
162 | 563k | static ExecEnv s_exec_env; |
163 | 563k | return &s_exec_env; |
164 | 563k | } |
165 | | |
166 | | // Requires ExenEnv ready |
167 | | static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id, |
168 | | SyncRowsetStats* sync_stats = nullptr, |
169 | | bool force_use_cache = false); |
170 | | |
171 | 453k | static bool ready() { return _s_ready.load(std::memory_order_acquire); } |
172 | 72 | static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); } |
173 | 5 | static bool get_is_upgrading() { return _s_upgrading.load(std::memory_order_acquire); } |
174 | 0 | static void set_is_upgrading() { _s_upgrading = true; } |
175 | | const std::string& token() const; |
176 | 0 | ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; } |
177 | 8 | vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; } |
178 | 0 | ResultBufferMgr* result_mgr() { return _result_mgr; } |
179 | 1 | ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; } |
180 | 0 | ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; } |
181 | 0 | ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; } |
182 | 0 | ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; } |
183 | | |
184 | 2.10k | WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; } |
185 | 0 | WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; } |
186 | 0 | RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() { |
187 | 0 | return _runtime_query_statistics_mgr; |
188 | 0 | } |
189 | | |
190 | | // using template to simplify client cache management |
191 | | template <typename T> |
192 | | inline ClientCache<T>* get_client_cache() { |
193 | | return nullptr; |
194 | | } |
195 | | |
196 | | // Save all MemTrackerLimiters in use. |
197 | | // Each group corresponds to several MemTrackerLimiters and has a lock. |
198 | | // Multiple groups are used to reduce the impact of locks. |
199 | | std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool; |
200 | | void init_mem_tracker(); |
201 | 182k | std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; } |
202 | 0 | std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() { |
203 | 0 | return _brpc_iobuf_block_memory_tracker; |
204 | 0 | } |
205 | 0 | std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() { |
206 | 0 | return _segcompaction_mem_tracker; |
207 | 0 | } |
208 | 8 | std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() { |
209 | 8 | return _stream_load_pipe_tracker; |
210 | 8 | } |
211 | 0 | std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() { |
212 | 0 | return _tablets_no_cache_mem_tracker; |
213 | 0 | } |
214 | 0 | std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() { |
215 | 0 | return _rowsets_no_cache_mem_tracker; |
216 | 0 | } |
217 | 0 | std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() { |
218 | 0 | return _segments_no_cache_mem_tracker; |
219 | 0 | } |
220 | 2.09k | std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() { |
221 | 2.09k | return _point_query_executor_mem_tracker; |
222 | 2.09k | } |
223 | 12 | std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() { |
224 | 12 | return _query_cache_mem_tracker; |
225 | 12 | } |
226 | 23.2k | std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() { |
227 | 23.2k | return _block_compression_mem_tracker; |
228 | 23.2k | } |
229 | 0 | std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() { |
230 | 0 | return _rowid_storage_reader_tracker; |
231 | 0 | } |
232 | 3.10k | std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() { |
233 | 3.10k | return _subcolumns_tree_tracker; |
234 | 3.10k | } |
235 | 542 | std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; } |
236 | 134 | std::shared_ptr<MemTrackerLimiter> parquet_meta_tracker() { return _parquet_meta_tracker; } |
237 | | |
238 | 0 | ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); } |
239 | 4 | ThreadPool* buffered_reader_prefetch_thread_pool() { |
240 | 4 | return _buffered_reader_prefetch_thread_pool.get(); |
241 | 4 | } |
242 | 0 | ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); } |
243 | 266 | ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); } |
244 | 0 | ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); } |
245 | | ThreadPool* non_block_close_thread_pool(); |
246 | 0 | ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); } |
247 | | |
248 | | void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths); |
249 | 128 | io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; } |
250 | 0 | UserFunctionCache* user_function_cache() { return _user_function_cache; } |
251 | 21 | FragmentMgr* fragment_mgr() { return _fragment_mgr; } |
252 | 0 | ResultCache* result_cache() { return _result_cache; } |
253 | 12 | ClusterInfo* cluster_info() { return _cluster_info; } |
254 | 1 | LoadPathMgr* load_path_mgr() { return _load_path_mgr; } |
255 | 1 | BfdParser* bfd_parser() const { return _bfd_parser; } |
256 | 0 | BrokerMgr* broker_mgr() const { return _broker_mgr; } |
257 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const { |
258 | 0 | return _internal_client_cache; |
259 | 0 | } |
260 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const { |
261 | 0 | return _streaming_client_cache; |
262 | 0 | } |
263 | 0 | BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const { |
264 | 0 | return _function_client_cache; |
265 | 0 | } |
266 | 0 | LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; } |
267 | 0 | LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); } |
268 | 3 | NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); } |
269 | 0 | SmallFileMgr* small_file_mgr() { return _small_file_mgr; } |
270 | 618 | doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; } |
271 | 0 | GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; } |
272 | | |
273 | 3 | const std::vector<StorePath>& store_paths() const { return _store_paths; } |
274 | | |
275 | 1 | StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); } |
276 | 0 | RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; } |
277 | 0 | HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; } |
278 | 14 | vectorized::ScannerScheduler* scanner_scheduler() { return _scanner_scheduler; } |
279 | 14 | FileMetaCache* file_meta_cache() { return _file_meta_cache; } |
280 | 16 | MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); } |
281 | 47 | WalManager* wal_mgr() { return _wal_manager.get(); } |
282 | 8 | DNSCache* dns_cache() { return _dns_cache; } |
283 | 5 | WriteCooldownMetaExecutors* write_cooldown_meta_executors() { |
284 | 5 | return _write_cooldown_meta_executors.get(); |
285 | 5 | } |
286 | | |
287 | 0 | kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } |
288 | 0 | io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } |
289 | 0 | IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; } |
290 | | |
291 | | #ifdef BE_TEST |
292 | 207 | void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) { |
293 | 207 | this->_tmp_file_dirs = std::move(tmp_file_dirs); |
294 | 207 | } |
295 | 4 | void set_ready() { this->_s_ready = true; } |
296 | 0 | void set_not_ready() { this->_s_ready = false; } |
297 | 36 | void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { |
298 | 36 | _memtable_memory_limiter.reset(limiter); |
299 | 36 | } |
300 | 5 | void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } |
301 | | void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr); |
302 | | void clear_new_load_stream_mgr(); |
303 | | void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor); |
304 | | void clear_stream_load_executor(); |
305 | | |
306 | | void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine); |
307 | | void set_inverted_index_searcher_cache( |
308 | | segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); |
309 | 1 | void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } |
310 | 1 | void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } |
311 | 1 | void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } |
312 | 1 | void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } |
313 | 1 | void set_tablet_column_object_pool(TabletColumnObjectPool* c) { |
314 | 1 | this->_tablet_column_object_pool = c; |
315 | 1 | } |
316 | 1 | void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } |
317 | 1 | void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } |
318 | 0 | void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { |
319 | 0 | this->_routine_load_task_executor = r; |
320 | 0 | } |
321 | | void set_wal_mgr(std::unique_ptr<WalManager>&& wm); |
322 | | void clear_wal_mgr(); |
323 | | |
324 | | void set_write_cooldown_meta_executors(); |
325 | 1 | static void set_tracking_memory(bool tracking_memory) { |
326 | 1 | _s_tracking_memory.store(tracking_memory, std::memory_order_release); |
327 | 1 | } |
328 | 1 | void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } |
329 | 2 | void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
330 | 2 | _non_block_close_thread_pool = std::move(pool); |
331 | 2 | } |
332 | | #endif |
333 | 0 | LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); } |
334 | | |
335 | 0 | vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); } |
336 | | |
337 | | void wait_for_all_tasks_done(); |
338 | | |
339 | | void update_frontends(const std::vector<TFrontendInfo>& new_infos); |
340 | | std::vector<TFrontendInfo> get_frontends(); |
341 | | std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); |
342 | | |
343 | 7.53k | TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } |
344 | 938 | TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } |
345 | 0 | SchemaCache* schema_cache() { return _schema_cache; } |
346 | 43.8k | StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } |
347 | 27.9k | SegmentLoader* segment_loader() { return _segment_loader; } |
348 | 0 | LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; } |
349 | 0 | RowCache* get_row_cache() { return _row_cache; } |
350 | 1.65k | CacheManager* get_cache_manager() { return _cache_manager; } |
351 | 0 | IdManager* get_id_manager() { return _id_manager; } |
352 | 1 | ProcessProfile* get_process_profile() { return _process_profile; } |
353 | 0 | HeapProfiler* get_heap_profiler() { return _heap_profiler; } |
354 | 259 | segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() { |
355 | 259 | return _inverted_index_searcher_cache; |
356 | 259 | } |
357 | 188 | segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() { |
358 | 188 | return _inverted_index_query_cache; |
359 | 188 | } |
360 | 3 | QueryCache* get_query_cache() { return _query_cache; } |
361 | | |
362 | 1 | pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { |
363 | 1 | return _runtime_filter_timer_queue; |
364 | 1 | } |
365 | | |
366 | 2 | vectorized::DictionaryFactory* dict_factory() { return _dict_factory; } |
367 | | |
368 | 0 | pipeline::PipelineTracerContext* pipeline_tracer_context() { |
369 | 0 | return _pipeline_tracer_ctx.get(); |
370 | 0 | } |
371 | | |
372 | 540 | segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); } |
373 | 3.79k | io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); } |
374 | | |
375 | 32 | orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; } |
376 | 0 | arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; } |
377 | | |
378 | | bool check_auth_token(const std::string& auth_token); |
379 | 3 | void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; } |
380 | | void clear_stream_mgr(); |
381 | | |
382 | 190 | DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } |
383 | | |
384 | | private: |
385 | | ExecEnv(); |
386 | | |
387 | | [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths, |
388 | | const std::vector<StorePath>& spill_store_paths, |
389 | | const std::set<std::string>& broken_paths); |
390 | | void _destroy(); |
391 | | |
392 | | Status _init_mem_env(); |
393 | | Status _check_deploy_mode(); |
394 | | |
395 | | Status _create_internal_workload_group(); |
396 | | void _init_runtime_filter_timer_queue(); |
397 | | |
398 | | inline static std::atomic_bool _s_ready {false}; |
399 | | inline static std::atomic_bool _s_tracking_memory {false}; |
400 | | std::vector<StorePath> _store_paths; |
401 | | std::vector<StorePath> _spill_store_paths; |
402 | | inline static std::atomic_bool _s_upgrading {false}; |
403 | | |
404 | | io::FileCacheFactory* _file_cache_factory = nullptr; |
405 | | UserFunctionCache* _user_function_cache = nullptr; |
406 | | // Leave protected so that subclasses can override |
407 | | ExternalScanContextMgr* _external_scan_context_mgr = nullptr; |
408 | | vectorized::VDataStreamMgr* _vstream_mgr = nullptr; |
409 | | ResultBufferMgr* _result_mgr = nullptr; |
410 | | ResultQueueMgr* _result_queue_mgr = nullptr; |
411 | | ClientCache<BackendServiceClient>* _backend_client_cache = nullptr; |
412 | | ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr; |
413 | | ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr; |
414 | | |
415 | | // The default tracker consumed by mem hook. If the thread does not attach other trackers, |
416 | | // by default all consumption will be passed to the process tracker through the orphan tracker. |
417 | | // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`. |
418 | | // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership", |
419 | | // and the consumption of the orphan mem tracker is close to 0, but greater than 0. |
420 | | std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker; |
421 | | std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker; |
422 | | // Count the memory consumption of segment compaction tasks. |
423 | | std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker; |
424 | | std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker; |
425 | | |
426 | | std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker; |
427 | | std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker; |
428 | | std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker; |
429 | | |
430 | | // Tracking memory may be shared between multiple queries. |
431 | | std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker; |
432 | | std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker; |
433 | | std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker; |
434 | | |
435 | | // TODO, looking forward to more accurate tracking. |
436 | | std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker; |
437 | | std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker; |
438 | | std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker; |
439 | | |
440 | | // Tracking memory consumption of parquet meta |
441 | | std::shared_ptr<MemTrackerLimiter> _parquet_meta_tracker; |
442 | | |
443 | | std::unique_ptr<ThreadPool> _send_batch_thread_pool; |
444 | | // Threadpool used to prefetch remote file for buffered reader |
445 | | std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool; |
446 | | // Threadpool used to send TableStats to FE |
447 | | std::unique_ptr<ThreadPool> _send_table_stats_thread_pool; |
448 | | // Threadpool used to upload local file to s3 |
449 | | std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool; |
450 | | // Pool used by join node to build hash table |
451 | | // Pool to use a new thread to release object |
452 | | std::unique_ptr<ThreadPool> _lazy_release_obj_pool; |
453 | | std::unique_ptr<ThreadPool> _non_block_close_thread_pool; |
454 | | std::unique_ptr<ThreadPool> _s3_file_system_thread_pool; |
455 | | |
456 | | FragmentMgr* _fragment_mgr = nullptr; |
457 | | WorkloadGroupMgr* _workload_group_manager = nullptr; |
458 | | |
459 | | ResultCache* _result_cache = nullptr; |
460 | | ClusterInfo* _cluster_info = nullptr; |
461 | | LoadPathMgr* _load_path_mgr = nullptr; |
462 | | |
463 | | BfdParser* _bfd_parser = nullptr; |
464 | | BrokerMgr* _broker_mgr = nullptr; |
465 | | LoadChannelMgr* _load_channel_mgr = nullptr; |
466 | | std::unique_ptr<LoadStreamMgr> _load_stream_mgr; |
467 | | std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr; |
468 | | BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr; |
469 | | BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr; |
470 | | BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr; |
471 | | |
472 | | std::unique_ptr<StreamLoadExecutor> _stream_load_executor; |
473 | | RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr; |
474 | | SmallFileMgr* _small_file_mgr = nullptr; |
475 | | HeartbeatFlags* _heartbeat_flags = nullptr; |
476 | | vectorized::ScannerScheduler* _scanner_scheduler = nullptr; |
477 | | |
478 | | // To save meta info of external file, such as parquet footer. |
479 | | FileMetaCache* _file_meta_cache = nullptr; |
480 | | std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter; |
481 | | std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool; |
482 | | std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool; |
483 | | std::unique_ptr<WalManager> _wal_manager; |
484 | | DNSCache* _dns_cache = nullptr; |
485 | | std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors; |
486 | | |
487 | | std::mutex _frontends_lock; |
488 | | // ip:brpc_port -> frontend_indo |
489 | | std::map<TNetworkAddress, FrontendInfo> _frontends; |
490 | | GroupCommitMgr* _group_commit_mgr = nullptr; |
491 | | |
492 | | // Maybe we should use unique_ptr, but it need complete type, which means we need |
493 | | // to include many headers, and for some cpp file that do not need class like TabletSchemaCache, |
494 | | // these redundancy header could introduce potential bug, at least, more header means slow compile. |
495 | | // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. |
496 | | TabletSchemaCache* _tablet_schema_cache = nullptr; |
497 | | TabletColumnObjectPool* _tablet_column_object_pool = nullptr; |
498 | | std::unique_ptr<BaseStorageEngine> _storage_engine; |
499 | | SchemaCache* _schema_cache = nullptr; |
500 | | StoragePageCache* _storage_page_cache = nullptr; |
501 | | SegmentLoader* _segment_loader = nullptr; |
502 | | LookupConnectionCache* _lookup_connection_cache = nullptr; |
503 | | RowCache* _row_cache = nullptr; |
504 | | CacheManager* _cache_manager = nullptr; |
505 | | IdManager* _id_manager = nullptr; |
506 | | ProcessProfile* _process_profile = nullptr; |
507 | | HeapProfiler* _heap_profiler = nullptr; |
508 | | segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr; |
509 | | segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr; |
510 | | QueryCache* _query_cache = nullptr; |
511 | | std::unique_ptr<io::FDCache> _file_cache_open_fd_cache; |
512 | | DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; |
513 | | |
514 | | pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; |
515 | | vectorized::DictionaryFactory* _dict_factory = nullptr; |
516 | | |
517 | | WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; |
518 | | IndexPolicyMgr* _index_policy_mgr = nullptr; |
519 | | |
520 | | RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr; |
521 | | |
522 | | std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx; |
523 | | std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs; |
524 | | doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr; |
525 | | |
526 | | orc::MemoryPool* _orc_memory_pool = nullptr; |
527 | | arrow::MemoryPool* _arrow_memory_pool = nullptr; |
528 | | |
529 | | kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; |
530 | | io::HdfsMgr* _hdfs_mgr = nullptr; |
531 | | }; |
532 | | |
533 | | template <> |
534 | 0 | inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() { |
535 | 0 | return _backend_client_cache; |
536 | 0 | } |
537 | | template <> |
538 | 0 | inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() { |
539 | 0 | return _frontend_client_cache; |
540 | 0 | } |
541 | | template <> |
542 | | inline ClientCache<TPaloBrokerServiceClient>* |
543 | 0 | ExecEnv::get_client_cache<TPaloBrokerServiceClient>() { |
544 | 0 | return _broker_client_cache; |
545 | 0 | } |
546 | | |
547 | 0 | inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() { |
548 | 0 | return ExecEnv::GetInstance()->get_inverted_index_query_cache(); |
549 | 0 | } |
550 | | |
551 | | } // namespace doris |