/root/doris/be/src/runtime/exec_env.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <common/multi_version.h> |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <map> |
25 | | #include <memory> |
26 | | #include <mutex> |
27 | | #include <optional> |
28 | | #include <string> |
29 | | #include <vector> |
30 | | |
31 | | #include "common/config.h" |
32 | | #include "common/status.h" |
33 | | #include "io/cache/fs_file_cache_storage.h" |
34 | | #include "olap/memtable_memory_limiter.h" |
35 | | #include "olap/options.h" |
36 | | #include "olap/rowset/segment_v2/inverted_index_writer.h" |
37 | | #include "olap/tablet_fwd.h" |
38 | | #include "pipeline/pipeline_tracing.h" |
39 | | #include "runtime/cluster_info.h" |
40 | | #include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header |
41 | | #include "util/threadpool.h" |
42 | | |
43 | | namespace orc { |
44 | | class MemoryPool; |
45 | | } |
46 | | namespace arrow { |
47 | | class MemoryPool; |
48 | | } |
49 | | |
50 | | namespace doris { |
51 | | namespace vectorized { |
52 | | class VDataStreamMgr; |
53 | | class SpillStreamManager; |
54 | | class DeltaWriterV2Pool; |
55 | | class DictionaryFactory; |
56 | | } // namespace vectorized |
57 | | namespace pipeline { |
58 | | class TaskScheduler; |
59 | | struct RuntimeFilterTimerQueue; |
60 | | } // namespace pipeline |
61 | | class WorkloadGroupMgr; |
62 | | struct WriteCooldownMetaExecutors; |
63 | | namespace io { |
64 | | class FileCacheFactory; |
65 | | class HdfsMgr; |
66 | | } // namespace io |
67 | | namespace segment_v2 { |
68 | | class InvertedIndexSearcherCache; |
69 | | class InvertedIndexQueryCache; |
70 | | class TmpFileDirs; |
71 | | |
72 | | namespace inverted_index { |
73 | | class AnalysisFactoryMgr; |
74 | | } |
75 | | |
76 | | } // namespace segment_v2 |
77 | | |
78 | | namespace kerberos { |
79 | | class KerberosTicketMgr; |
80 | | } |
81 | | |
82 | | class QueryCache; |
83 | | class WorkloadSchedPolicyMgr; |
84 | | class BfdParser; |
85 | | class BrokerMgr; |
86 | | template <class T> |
87 | | class BrpcClientCache; |
88 | | class ExternalScanContextMgr; |
89 | | class FragmentMgr; |
90 | | class ResultCache; |
91 | | class LoadPathMgr; |
92 | | class NewLoadStreamMgr; |
93 | | class MemTrackerLimiter; |
94 | | class MemTracker; |
95 | | struct TrackerLimiterGroup; |
96 | | class BaseStorageEngine; |
97 | | class ResultBufferMgr; |
98 | | class ResultQueueMgr; |
99 | | class RuntimeQueryStatisticsMgr; |
100 | | class LoadChannelMgr; |
101 | | class LoadStreamMgr; |
102 | | class LoadStreamMapPool; |
103 | | class StreamLoadExecutor; |
104 | | class RoutineLoadTaskExecutor; |
105 | | class SmallFileMgr; |
106 | | class BackendServiceClient; |
107 | | class TPaloBrokerServiceClient; |
108 | | class PBackendService_Stub; |
109 | | class PFunctionService_Stub; |
110 | | template <class T> |
111 | | class ClientCache; |
112 | | class HeartbeatFlags; |
113 | | class FrontendServiceClient; |
114 | | class FileMetaCache; |
115 | | class GroupCommitMgr; |
116 | | class TabletSchemaCache; |
117 | | class TabletColumnObjectPool; |
118 | | class UserFunctionCache; |
119 | | class SchemaCache; |
120 | | class StoragePageCache; |
121 | | class SegmentLoader; |
122 | | class LookupConnectionCache; |
123 | | class RowCache; |
124 | | class DummyLRUCache; |
125 | | class CacheManager; |
126 | | class IdManager; |
127 | | class ProcessProfile; |
128 | | class HeapProfiler; |
129 | | class WalManager; |
130 | | class DNSCache; |
131 | | class IndexPolicyMgr; |
132 | | struct SyncRowsetStats; |
133 | | class DeleteBitmapAggCache; |
134 | | |
135 | | inline bool k_doris_exit = false; |
136 | | |
137 | | // Execution environment for queries/plan fragments. |
138 | | // Contains all required global structures, and handles to |
139 | | // singleton services. Clients must call StartServices exactly |
140 | | // once to properly initialise service state. |
141 | | class ExecEnv { |
142 | | public: |
143 | | // Empty destructor because the compiler-generated one requires full |
144 | | // declarations for classes in scoped_ptrs. |
145 | | ~ExecEnv(); |
146 | | |
147 | 8.92k | BaseStorageEngine& storage_engine() { return *_storage_engine; } |
148 | | |
149 | | // Initial exec environment. must call this to init all |
150 | | [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths, |
151 | | const std::vector<StorePath>& spill_store_paths, |
152 | | const std::set<std::string>& broken_paths); |
153 | | |
154 | | // Stop all threads and delete resources. |
155 | | void destroy(); |
156 | | |
157 | | /// Returns the first created exec env instance. In a normal doris, this is |
158 | | /// the only instance. In test setups with multiple ExecEnv's per process, |
159 | | /// we return the most recently created instance. |
160 | 573k | static ExecEnv* GetInstance() { |
161 | 573k | static ExecEnv s_exec_env; |
162 | 573k | return &s_exec_env; |
163 | 573k | } |
164 | | |
165 | | // Requires ExenEnv ready |
166 | | static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id, |
167 | | SyncRowsetStats* sync_stats = nullptr, |
168 | | bool force_use_only_cached = false); |
169 | | |
170 | 452k | static bool ready() { return _s_ready.load(std::memory_order_acquire); } |
171 | 72 | static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); } |
172 | 5 | static bool get_is_upgrading() { return _s_upgrading.load(std::memory_order_acquire); } |
173 | 0 | static void set_is_upgrading() { _s_upgrading = true; } |
174 | | const std::string& token() const; |
175 | 0 | ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; } |
176 | 8 | vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; } |
177 | 0 | ResultBufferMgr* result_mgr() { return _result_mgr; } |
178 | 1 | ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; } |
179 | 0 | ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; } |
180 | 0 | ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; } |
181 | 0 | ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; } |
182 | | |
183 | 2.16k | WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; } |
184 | 0 | WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; } |
185 | 0 | RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() { |
186 | 0 | return _runtime_query_statistics_mgr; |
187 | 0 | } |
188 | | |
189 | | // using template to simplify client cache management |
190 | | template <typename T> |
191 | | inline ClientCache<T>* get_client_cache() { |
192 | | return nullptr; |
193 | | } |
194 | | |
195 | | // Save all MemTrackerLimiters in use. |
196 | | // Each group corresponds to several MemTrackerLimiters and has a lock. |
197 | | // Multiple groups are used to reduce the impact of locks. |
198 | | std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool; |
199 | | void init_mem_tracker(); |
200 | 189k | std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; } |
201 | 0 | std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() { |
202 | 0 | return _brpc_iobuf_block_memory_tracker; |
203 | 0 | } |
204 | 0 | std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() { |
205 | 0 | return _segcompaction_mem_tracker; |
206 | 0 | } |
207 | 8 | std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() { |
208 | 8 | return _stream_load_pipe_tracker; |
209 | 8 | } |
210 | 0 | std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() { |
211 | 0 | return _tablets_no_cache_mem_tracker; |
212 | 0 | } |
213 | 0 | std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() { |
214 | 0 | return _rowsets_no_cache_mem_tracker; |
215 | 0 | } |
216 | 0 | std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() { |
217 | 0 | return _segments_no_cache_mem_tracker; |
218 | 0 | } |
219 | 2.09k | std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() { |
220 | 2.09k | return _point_query_executor_mem_tracker; |
221 | 2.09k | } |
222 | 12 | std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() { |
223 | 12 | return _query_cache_mem_tracker; |
224 | 12 | } |
225 | 23.2k | std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() { |
226 | 23.2k | return _block_compression_mem_tracker; |
227 | 23.2k | } |
228 | 0 | std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() { |
229 | 0 | return _rowid_storage_reader_tracker; |
230 | 0 | } |
231 | 3.10k | std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() { |
232 | 3.10k | return _subcolumns_tree_tracker; |
233 | 3.10k | } |
234 | 676 | std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; } |
235 | 134 | std::shared_ptr<MemTrackerLimiter> parquet_meta_tracker() { return _parquet_meta_tracker; } |
236 | | |
237 | 0 | ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); } |
238 | 4 | ThreadPool* buffered_reader_prefetch_thread_pool() { |
239 | 4 | return _buffered_reader_prefetch_thread_pool.get(); |
240 | 4 | } |
241 | 0 | ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); } |
242 | 333 | ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); } |
243 | 0 | ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); } |
244 | | ThreadPool* non_block_close_thread_pool(); |
245 | 0 | ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); } |
246 | 0 | ThreadPool* udf_close_workers_pool() { return _udf_close_workers_thread_pool.get(); } |
247 | | |
248 | | void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths); |
249 | 128 | io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; } |
250 | 0 | UserFunctionCache* user_function_cache() { return _user_function_cache; } |
251 | 21 | FragmentMgr* fragment_mgr() { return _fragment_mgr; } |
252 | 0 | ResultCache* result_cache() { return _result_cache; } |
253 | 15 | ClusterInfo* cluster_info() { return _cluster_info; } |
254 | 1 | LoadPathMgr* load_path_mgr() { return _load_path_mgr; } |
255 | 1 | BfdParser* bfd_parser() const { return _bfd_parser; } |
256 | 0 | BrokerMgr* broker_mgr() const { return _broker_mgr; } |
257 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const { |
258 | 0 | return _internal_client_cache; |
259 | 0 | } |
260 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const { |
261 | 0 | return _streaming_client_cache; |
262 | 0 | } |
263 | 0 | BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const { |
264 | 0 | return _function_client_cache; |
265 | 0 | } |
266 | 0 | LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; } |
267 | 0 | LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); } |
268 | 3 | NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); } |
269 | 0 | SmallFileMgr* small_file_mgr() { return _small_file_mgr; } |
270 | 621 | doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; } |
271 | 0 | GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; } |
272 | | |
273 | 3 | const std::vector<StorePath>& store_paths() const { return _store_paths; } |
274 | | |
275 | 1 | StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); } |
276 | 0 | RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; } |
277 | 0 | HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; } |
278 | 14 | FileMetaCache* file_meta_cache() { return _file_meta_cache; } |
279 | 16 | MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); } |
280 | 47 | WalManager* wal_mgr() { return _wal_manager.get(); } |
281 | 8 | DNSCache* dns_cache() { return _dns_cache; } |
282 | 5 | WriteCooldownMetaExecutors* write_cooldown_meta_executors() { |
283 | 5 | return _write_cooldown_meta_executors.get(); |
284 | 5 | } |
285 | | |
286 | 0 | kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } |
287 | 0 | io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } |
288 | 3 | IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; } |
289 | | |
290 | | #ifdef BE_TEST |
291 | 210 | void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) { |
292 | 210 | this->_tmp_file_dirs = std::move(tmp_file_dirs); |
293 | 210 | } |
294 | 4 | void set_ready() { this->_s_ready = true; } |
295 | 0 | void set_not_ready() { this->_s_ready = false; } |
296 | 36 | void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { |
297 | 36 | _memtable_memory_limiter.reset(limiter); |
298 | 36 | } |
299 | 5 | void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } |
300 | | void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr); |
301 | | void clear_new_load_stream_mgr(); |
302 | | void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor); |
303 | | void clear_stream_load_executor(); |
304 | | |
305 | | void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine); |
306 | | void set_inverted_index_searcher_cache( |
307 | | segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); |
308 | 1 | void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } |
309 | 1 | void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } |
310 | 1 | void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } |
311 | 1 | void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } |
312 | 1 | void set_tablet_column_object_pool(TabletColumnObjectPool* c) { |
313 | 1 | this->_tablet_column_object_pool = c; |
314 | 1 | } |
315 | 1 | void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } |
316 | 1 | void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } |
317 | 0 | void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { |
318 | 0 | this->_routine_load_task_executor = r; |
319 | 0 | } |
320 | | void set_wal_mgr(std::unique_ptr<WalManager>&& wm); |
321 | | void clear_wal_mgr(); |
322 | | |
323 | | void set_write_cooldown_meta_executors(); |
324 | 1 | static void set_tracking_memory(bool tracking_memory) { |
325 | 1 | _s_tracking_memory.store(tracking_memory, std::memory_order_release); |
326 | 1 | } |
327 | 1 | void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } |
328 | 2 | void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
329 | 2 | _non_block_close_thread_pool = std::move(pool); |
330 | 2 | } |
331 | 0 | void set_s3_file_upload_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
332 | 0 | _s3_file_upload_thread_pool = std::move(pool); |
333 | 0 | } |
334 | 0 | void set_file_cache_factory(io::FileCacheFactory* factory) { _file_cache_factory = factory; } |
335 | 0 | void set_file_cache_open_fd_cache(std::unique_ptr<io::FDCache>&& fd_cache) { |
336 | 0 | _file_cache_open_fd_cache = std::move(fd_cache); |
337 | 0 | } |
338 | | #endif |
339 | 0 | LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); } |
340 | | |
341 | 0 | vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); } |
342 | | |
343 | | void wait_for_all_tasks_done(); |
344 | | |
345 | | void update_frontends(const std::vector<TFrontendInfo>& new_infos); |
346 | | std::vector<TFrontendInfo> get_frontends(); |
347 | | std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); |
348 | | |
349 | 7.72k | TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } |
350 | 938 | TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } |
351 | 0 | SchemaCache* schema_cache() { return _schema_cache; } |
352 | 43.9k | StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } |
353 | 27.9k | SegmentLoader* segment_loader() { return _segment_loader; } |
354 | 0 | LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; } |
355 | 0 | RowCache* get_row_cache() { return _row_cache; } |
356 | 1.78k | CacheManager* get_cache_manager() { return _cache_manager; } |
357 | 0 | IdManager* get_id_manager() { return _id_manager; } |
358 | 1 | ProcessProfile* get_process_profile() { return _process_profile; } |
359 | 0 | HeapProfiler* get_heap_profiler() { return _heap_profiler; } |
360 | 259 | segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() { |
361 | 259 | return _inverted_index_searcher_cache; |
362 | 259 | } |
363 | 190 | segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() { |
364 | 190 | return _inverted_index_query_cache; |
365 | 190 | } |
366 | 3 | QueryCache* get_query_cache() { return _query_cache; } |
367 | | |
368 | 1 | pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { |
369 | 1 | return _runtime_filter_timer_queue; |
370 | 1 | } |
371 | | |
372 | 2 | vectorized::DictionaryFactory* dict_factory() { return _dict_factory; } |
373 | | |
374 | 0 | pipeline::PipelineTracerContext* pipeline_tracer_context() { |
375 | 0 | return _pipeline_tracer_ctx.get(); |
376 | 0 | } |
377 | | |
378 | 551 | segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); } |
379 | 3.77k | io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); } |
380 | | |
381 | 32 | orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; } |
382 | 0 | arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; } |
383 | | |
384 | | bool check_auth_token(const std::string& auth_token); |
385 | 3 | void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; } |
386 | | void clear_stream_mgr(); |
387 | | |
388 | 190 | DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } |
389 | | Status init_mem_env(); |
390 | | |
391 | | private: |
392 | | ExecEnv(); |
393 | | |
394 | | [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths, |
395 | | const std::vector<StorePath>& spill_store_paths, |
396 | | const std::set<std::string>& broken_paths); |
397 | | void _destroy(); |
398 | | |
399 | | Status _check_deploy_mode(); |
400 | | |
401 | | Status _create_internal_workload_group(); |
402 | | void _init_runtime_filter_timer_queue(); |
403 | | |
404 | | inline static std::atomic_bool _s_ready {false}; |
405 | | inline static std::atomic_bool _s_tracking_memory {false}; |
406 | | std::vector<StorePath> _store_paths; |
407 | | std::vector<StorePath> _spill_store_paths; |
408 | | inline static std::atomic_bool _s_upgrading {false}; |
409 | | |
410 | | io::FileCacheFactory* _file_cache_factory = nullptr; |
411 | | UserFunctionCache* _user_function_cache = nullptr; |
412 | | // Leave protected so that subclasses can override |
413 | | ExternalScanContextMgr* _external_scan_context_mgr = nullptr; |
414 | | vectorized::VDataStreamMgr* _vstream_mgr = nullptr; |
415 | | ResultBufferMgr* _result_mgr = nullptr; |
416 | | ResultQueueMgr* _result_queue_mgr = nullptr; |
417 | | ClientCache<BackendServiceClient>* _backend_client_cache = nullptr; |
418 | | ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr; |
419 | | ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr; |
420 | | |
421 | | // The default tracker consumed by mem hook. If the thread does not attach other trackers, |
422 | | // by default all consumption will be passed to the process tracker through the orphan tracker. |
423 | | // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`. |
424 | | // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership", |
425 | | // and the consumption of the orphan mem tracker is close to 0, but greater than 0. |
426 | | std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker; |
427 | | std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker; |
428 | | // Count the memory consumption of segment compaction tasks. |
429 | | std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker; |
430 | | std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker; |
431 | | |
432 | | std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker; |
433 | | std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker; |
434 | | std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker; |
435 | | |
436 | | // Tracking memory may be shared between multiple queries. |
437 | | std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker; |
438 | | std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker; |
439 | | std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker; |
440 | | |
441 | | // TODO, looking forward to more accurate tracking. |
442 | | std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker; |
443 | | std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker; |
444 | | std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker; |
445 | | |
446 | | // Tracking memory consumption of parquet meta |
447 | | std::shared_ptr<MemTrackerLimiter> _parquet_meta_tracker; |
448 | | |
449 | | std::unique_ptr<ThreadPool> _send_batch_thread_pool; |
450 | | // Threadpool used to prefetch remote file for buffered reader |
451 | | std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool; |
452 | | // Threadpool used to send TableStats to FE |
453 | | std::unique_ptr<ThreadPool> _send_table_stats_thread_pool; |
454 | | // Threadpool used to upload local file to s3 |
455 | | std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool; |
456 | | // Pool used by join node to build hash table |
457 | | // Pool to use a new thread to release object |
458 | | std::unique_ptr<ThreadPool> _lazy_release_obj_pool; |
459 | | std::unique_ptr<ThreadPool> _non_block_close_thread_pool; |
460 | | std::unique_ptr<ThreadPool> _s3_file_system_thread_pool; |
461 | | // for java-udf to close |
462 | | std::unique_ptr<ThreadPool> _udf_close_workers_thread_pool; |
463 | | |
464 | | FragmentMgr* _fragment_mgr = nullptr; |
465 | | WorkloadGroupMgr* _workload_group_manager = nullptr; |
466 | | |
467 | | ResultCache* _result_cache = nullptr; |
468 | | ClusterInfo* _cluster_info = nullptr; |
469 | | LoadPathMgr* _load_path_mgr = nullptr; |
470 | | |
471 | | BfdParser* _bfd_parser = nullptr; |
472 | | BrokerMgr* _broker_mgr = nullptr; |
473 | | LoadChannelMgr* _load_channel_mgr = nullptr; |
474 | | std::unique_ptr<LoadStreamMgr> _load_stream_mgr; |
475 | | std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr; |
476 | | BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr; |
477 | | BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr; |
478 | | BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr; |
479 | | |
480 | | std::unique_ptr<StreamLoadExecutor> _stream_load_executor; |
481 | | RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr; |
482 | | SmallFileMgr* _small_file_mgr = nullptr; |
483 | | HeartbeatFlags* _heartbeat_flags = nullptr; |
484 | | |
485 | | // To save meta info of external file, such as parquet footer. |
486 | | FileMetaCache* _file_meta_cache = nullptr; |
487 | | std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter; |
488 | | std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool; |
489 | | std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool; |
490 | | std::unique_ptr<WalManager> _wal_manager; |
491 | | DNSCache* _dns_cache = nullptr; |
492 | | std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors; |
493 | | |
494 | | std::mutex _frontends_lock; |
495 | | // ip:brpc_port -> frontend_indo |
496 | | std::map<TNetworkAddress, FrontendInfo> _frontends; |
497 | | GroupCommitMgr* _group_commit_mgr = nullptr; |
498 | | |
499 | | // Maybe we should use unique_ptr, but it need complete type, which means we need |
500 | | // to include many headers, and for some cpp file that do not need class like TabletSchemaCache, |
501 | | // these redundancy header could introduce potential bug, at least, more header means slow compile. |
502 | | // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. |
503 | | TabletSchemaCache* _tablet_schema_cache = nullptr; |
504 | | TabletColumnObjectPool* _tablet_column_object_pool = nullptr; |
505 | | std::unique_ptr<BaseStorageEngine> _storage_engine; |
506 | | SchemaCache* _schema_cache = nullptr; |
507 | | StoragePageCache* _storage_page_cache = nullptr; |
508 | | SegmentLoader* _segment_loader = nullptr; |
509 | | LookupConnectionCache* _lookup_connection_cache = nullptr; |
510 | | RowCache* _row_cache = nullptr; |
511 | | CacheManager* _cache_manager = nullptr; |
512 | | IdManager* _id_manager = nullptr; |
513 | | ProcessProfile* _process_profile = nullptr; |
514 | | HeapProfiler* _heap_profiler = nullptr; |
515 | | segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr; |
516 | | segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr; |
517 | | QueryCache* _query_cache = nullptr; |
518 | | std::unique_ptr<io::FDCache> _file_cache_open_fd_cache; |
519 | | DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; |
520 | | |
521 | | pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; |
522 | | vectorized::DictionaryFactory* _dict_factory = nullptr; |
523 | | |
524 | | WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; |
525 | | IndexPolicyMgr* _index_policy_mgr = nullptr; |
526 | | |
527 | | RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr; |
528 | | |
529 | | std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx; |
530 | | std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs; |
531 | | doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr; |
532 | | |
533 | | orc::MemoryPool* _orc_memory_pool = nullptr; |
534 | | arrow::MemoryPool* _arrow_memory_pool = nullptr; |
535 | | |
536 | | kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; |
537 | | io::HdfsMgr* _hdfs_mgr = nullptr; |
538 | | }; |
539 | | |
540 | | template <> |
541 | 0 | inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() { |
542 | 0 | return _backend_client_cache; |
543 | 0 | } |
544 | | template <> |
545 | 0 | inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() { |
546 | 0 | return _frontend_client_cache; |
547 | 0 | } |
548 | | template <> |
549 | | inline ClientCache<TPaloBrokerServiceClient>* |
550 | 0 | ExecEnv::get_client_cache<TPaloBrokerServiceClient>() { |
551 | 0 | return _broker_client_cache; |
552 | 0 | } |
553 | | |
554 | 0 | inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() { |
555 | 0 | return ExecEnv::GetInstance()->get_inverted_index_query_cache(); |
556 | 0 | } |
557 | | |
558 | | } // namespace doris |