/root/doris/be/src/runtime/exec_env.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <common/multi_version.h> |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <map> |
25 | | #include <memory> |
26 | | #include <mutex> |
27 | | #include <optional> |
28 | | #include <string> |
29 | | #include <vector> |
30 | | |
31 | | #include "common/config.h" |
32 | | #include "common/status.h" |
33 | | #include "exec/schema_scanner/schema_routine_load_job_scanner.h" |
34 | | #include "io/cache/fs_file_cache_storage.h" |
35 | | #include "olap/memtable_memory_limiter.h" |
36 | | #include "olap/options.h" |
37 | | #include "olap/rowset/segment_v2/inverted_index_writer.h" |
38 | | #include "olap/tablet_fwd.h" |
39 | | #include "pipeline/pipeline_tracing.h" |
40 | | #include "runtime/cluster_info.h" |
41 | | #include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header |
42 | | #include "util/threadpool.h" |
43 | | |
44 | | namespace orc { |
45 | | class MemoryPool; |
46 | | } |
47 | | namespace arrow { |
48 | | class MemoryPool; |
49 | | } |
50 | | |
51 | | namespace doris { |
52 | | namespace vectorized { |
53 | | class VDataStreamMgr; |
54 | | class ScannerScheduler; |
55 | | class SpillStreamManager; |
56 | | class DeltaWriterV2Pool; |
57 | | class DictionaryFactory; |
58 | | } // namespace vectorized |
59 | | namespace pipeline { |
60 | | class TaskScheduler; |
61 | | struct RuntimeFilterTimerQueue; |
62 | | } // namespace pipeline |
63 | | class WorkloadGroupMgr; |
64 | | struct WriteCooldownMetaExecutors; |
65 | | namespace io { |
66 | | class FileCacheFactory; |
67 | | class HdfsMgr; |
68 | | } // namespace io |
69 | | namespace segment_v2 { |
70 | | class InvertedIndexSearcherCache; |
71 | | class InvertedIndexQueryCache; |
72 | | class ConditionCache; |
73 | | class TmpFileDirs; |
74 | | class EncodingInfoResolver; |
75 | | |
76 | | namespace inverted_index { |
77 | | class AnalysisFactoryMgr; |
78 | | } |
79 | | |
80 | | } // namespace segment_v2 |
81 | | |
82 | | namespace kerberos { |
83 | | class KerberosTicketMgr; |
84 | | } |
85 | | |
86 | | class QueryCache; |
87 | | class WorkloadSchedPolicyMgr; |
88 | | class BfdParser; |
89 | | class BrokerMgr; |
90 | | template <class T> |
91 | | class BrpcClientCache; |
92 | | class ExternalScanContextMgr; |
93 | | class FragmentMgr; |
94 | | class ResultCache; |
95 | | class LoadPathMgr; |
96 | | class NewLoadStreamMgr; |
97 | | class MemTrackerLimiter; |
98 | | class MemTracker; |
99 | | struct TrackerLimiterGroup; |
100 | | class BaseStorageEngine; |
101 | | class ResultBufferMgr; |
102 | | class ResultQueueMgr; |
103 | | class RuntimeQueryStatisticsMgr; |
104 | | class LoadChannelMgr; |
105 | | class LoadStreamMgr; |
106 | | class LoadStreamMapPool; |
107 | | class StreamLoadExecutor; |
108 | | class RoutineLoadTaskExecutor; |
109 | | class StreamLoadRecorderManager; |
110 | | class SmallFileMgr; |
111 | | class BackendServiceClient; |
112 | | class TPaloBrokerServiceClient; |
113 | | class PBackendService_Stub; |
114 | | class PFunctionService_Stub; |
115 | | template <class T> |
116 | | class ClientCache; |
117 | | class HeartbeatFlags; |
118 | | class FrontendServiceClient; |
119 | | class FileMetaCache; |
120 | | class GroupCommitMgr; |
121 | | class TabletSchemaCache; |
122 | | class TabletColumnObjectPool; |
123 | | class UserFunctionCache; |
124 | | class SchemaCache; |
125 | | class StoragePageCache; |
126 | | class SegmentLoader; |
127 | | class LookupConnectionCache; |
128 | | class RowCache; |
129 | | class DummyLRUCache; |
130 | | class CacheManager; |
131 | | class IdManager; |
132 | | class ProcessProfile; |
133 | | class HeapProfiler; |
134 | | class WalManager; |
135 | | class DNSCache; |
136 | | class IndexPolicyMgr; |
137 | | struct SyncRowsetStats; |
138 | | class DeleteBitmapAggCache; |
139 | | |
140 | | // set to true when BE is shutting down |
141 | | inline bool k_doris_exit = false; |
142 | | // set to true after BE start ready |
143 | | inline bool k_is_server_ready = false; |
144 | | |
145 | | // Execution environment for queries/plan fragments. |
146 | | // Contains all required global structures, and handles to |
147 | | // singleton services. Clients must call StartServices exactly |
148 | | // once to properly initialise service state. |
149 | | class ExecEnv { |
150 | | public: |
151 | | // Empty destructor because the compiler-generated one requires full |
152 | | // declarations for classes in scoped_ptrs. |
153 | | ~ExecEnv(); |
154 | | |
155 | 8.92k | BaseStorageEngine& storage_engine() { return *_storage_engine; } |
156 | | |
157 | | // Initial exec environment. must call this to init all |
158 | | [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths, |
159 | | const std::vector<StorePath>& spill_store_paths, |
160 | | const std::set<std::string>& broken_paths); |
161 | | |
162 | | // Stop all threads and delete resources. |
163 | | void destroy(); |
164 | | |
165 | | /// Returns the first created exec env instance. In a normal doris, this is |
166 | | /// the only instance. In test setups with multiple ExecEnv's per process, |
167 | | /// we return the most recently created instance. |
168 | 691k | static ExecEnv* GetInstance() { |
169 | 691k | static ExecEnv s_exec_env; |
170 | 691k | return &s_exec_env; |
171 | 691k | } |
172 | | |
173 | | // Requires ExenEnv ready |
174 | | static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id, |
175 | | SyncRowsetStats* sync_stats = nullptr, |
176 | | bool force_use_only_cached = false); |
177 | | |
178 | 452k | static bool ready() { return _s_ready.load(std::memory_order_acquire); } |
179 | 72 | static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); } |
180 | 8 | static bool get_is_upgrading() { return _s_upgrading.load(std::memory_order_acquire); } |
181 | 0 | static void set_is_upgrading() { _s_upgrading = true; } |
182 | | const std::string& token() const; |
183 | 0 | ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; } |
184 | 8 | vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; } |
185 | 0 | ResultBufferMgr* result_mgr() { return _result_mgr; } |
186 | 1 | ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; } |
187 | 0 | ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; } |
188 | 0 | ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; } |
189 | 0 | ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; } |
190 | | |
191 | 2.16k | WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; } |
192 | 0 | WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; } |
193 | 0 | RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() { |
194 | 0 | return _runtime_query_statistics_mgr; |
195 | 0 | } |
196 | | |
197 | | // using template to simplify client cache management |
198 | | template <typename T> |
199 | | inline ClientCache<T>* get_client_cache() { |
200 | | return nullptr; |
201 | | } |
202 | | |
203 | | // Save all MemTrackerLimiters in use. |
204 | | // Each group corresponds to several MemTrackerLimiters and has a lock. |
205 | | // Multiple groups are used to reduce the impact of locks. |
206 | | std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool; |
207 | | void init_mem_tracker(); |
208 | 183k | std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; } |
209 | 0 | std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() { |
210 | 0 | return _brpc_iobuf_block_memory_tracker; |
211 | 0 | } |
212 | 0 | std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() { |
213 | 0 | return _segcompaction_mem_tracker; |
214 | 0 | } |
215 | 8 | std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() { |
216 | 8 | return _stream_load_pipe_tracker; |
217 | 8 | } |
218 | 0 | std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() { |
219 | 0 | return _tablets_no_cache_mem_tracker; |
220 | 0 | } |
221 | 0 | std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() { |
222 | 0 | return _rowsets_no_cache_mem_tracker; |
223 | 0 | } |
224 | 0 | std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() { |
225 | 0 | return _segments_no_cache_mem_tracker; |
226 | 0 | } |
227 | 2.09k | std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() { |
228 | 2.09k | return _point_query_executor_mem_tracker; |
229 | 2.09k | } |
230 | 12 | std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() { |
231 | 12 | return _query_cache_mem_tracker; |
232 | 12 | } |
233 | 23.2k | std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() { |
234 | 23.2k | return _block_compression_mem_tracker; |
235 | 23.2k | } |
236 | 0 | std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() { |
237 | 0 | return _rowid_storage_reader_tracker; |
238 | 0 | } |
239 | 2.60k | std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() { |
240 | 2.60k | return _subcolumns_tree_tracker; |
241 | 2.60k | } |
242 | 720 | std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; } |
243 | 162 | std::shared_ptr<MemTrackerLimiter> parquet_meta_tracker() { return _parquet_meta_tracker; } |
244 | | |
245 | 0 | ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); } |
246 | 4 | ThreadPool* buffered_reader_prefetch_thread_pool() { |
247 | 4 | return _buffered_reader_prefetch_thread_pool.get(); |
248 | 4 | } |
249 | 0 | ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); } |
250 | 355 | ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); } |
251 | 0 | ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); } |
252 | | ThreadPool* non_block_close_thread_pool(); |
253 | 0 | ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); } |
254 | 0 | ThreadPool* udf_close_workers_pool() { return _udf_close_workers_thread_pool.get(); } |
255 | | |
256 | | void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths); |
257 | 151 | io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; } |
258 | 0 | UserFunctionCache* user_function_cache() { return _user_function_cache; } |
259 | 124k | FragmentMgr* fragment_mgr() { return _fragment_mgr; } |
260 | 0 | ResultCache* result_cache() { return _result_cache; } |
261 | 15 | ClusterInfo* cluster_info() { return _cluster_info; } |
262 | 1 | LoadPathMgr* load_path_mgr() { return _load_path_mgr; } |
263 | 1 | BfdParser* bfd_parser() const { return _bfd_parser; } |
264 | 0 | BrokerMgr* broker_mgr() const { return _broker_mgr; } |
265 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const { |
266 | 0 | return _internal_client_cache; |
267 | 0 | } |
268 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const { |
269 | 0 | return _streaming_client_cache; |
270 | 0 | } |
271 | 0 | BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const { |
272 | 0 | return _function_client_cache; |
273 | 0 | } |
274 | 0 | LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; } |
275 | 0 | LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); } |
276 | 3 | NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); } |
277 | 0 | SmallFileMgr* small_file_mgr() { return _small_file_mgr; } |
278 | 615 | doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; } |
279 | 0 | GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; } |
280 | | |
281 | 3 | const std::vector<StorePath>& store_paths() const { return _store_paths; } |
282 | | |
283 | 1 | StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); } |
284 | 0 | RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; } |
285 | 0 | HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; } |
286 | 14 | vectorized::ScannerScheduler* scanner_scheduler() { return _scanner_scheduler; } |
287 | 14 | FileMetaCache* file_meta_cache() { return _file_meta_cache; } |
288 | 16 | MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); } |
289 | 47 | WalManager* wal_mgr() { return _wal_manager.get(); } |
290 | 8 | DNSCache* dns_cache() { return _dns_cache; } |
291 | 5 | WriteCooldownMetaExecutors* write_cooldown_meta_executors() { |
292 | 5 | return _write_cooldown_meta_executors.get(); |
293 | 5 | } |
294 | | |
295 | 0 | kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } |
296 | 0 | io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } |
297 | 3 | IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; } |
298 | | |
299 | | #ifdef BE_TEST |
300 | 213 | void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) { |
301 | 213 | this->_tmp_file_dirs = std::move(tmp_file_dirs); |
302 | 213 | } |
303 | 4 | void set_ready() { this->_s_ready = true; } |
304 | 0 | void set_not_ready() { this->_s_ready = false; } |
305 | 36 | void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { |
306 | 36 | _memtable_memory_limiter.reset(limiter); |
307 | 36 | } |
308 | 5 | void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } |
309 | | void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr); |
310 | | void clear_new_load_stream_mgr(); |
311 | | void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor); |
312 | | void clear_stream_load_executor(); |
313 | | |
314 | | void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine); |
315 | | void set_inverted_index_searcher_cache( |
316 | | segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); |
317 | 1 | void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } |
318 | 1 | void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } |
319 | 1 | void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } |
320 | 1 | void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } |
321 | 1 | void set_tablet_column_object_pool(TabletColumnObjectPool* c) { |
322 | 1 | this->_tablet_column_object_pool = c; |
323 | 1 | } |
324 | 1 | void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } |
325 | 1 | void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } |
326 | 0 | void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { |
327 | 0 | this->_routine_load_task_executor = r; |
328 | 0 | } |
329 | | void set_wal_mgr(std::unique_ptr<WalManager>&& wm); |
330 | | void clear_wal_mgr(); |
331 | | |
332 | | void set_write_cooldown_meta_executors(); |
333 | 1 | static void set_tracking_memory(bool tracking_memory) { |
334 | 1 | _s_tracking_memory.store(tracking_memory, std::memory_order_release); |
335 | 1 | } |
336 | 1 | void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } |
337 | 2 | void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
338 | 2 | _non_block_close_thread_pool = std::move(pool); |
339 | 2 | } |
340 | 0 | void set_s3_file_upload_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
341 | 0 | _s3_file_upload_thread_pool = std::move(pool); |
342 | 0 | } |
343 | 0 | void set_file_cache_factory(io::FileCacheFactory* factory) { _file_cache_factory = factory; } |
344 | 0 | void set_file_cache_open_fd_cache(std::unique_ptr<io::FDCache>&& fd_cache) { |
345 | 0 | _file_cache_open_fd_cache = std::move(fd_cache); |
346 | 0 | } |
347 | | #endif |
348 | 0 | LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); } |
349 | | |
350 | 0 | vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); } |
351 | | |
352 | | void wait_for_all_tasks_done(); |
353 | | |
354 | | void update_frontends(const std::vector<TFrontendInfo>& new_infos); |
355 | | std::vector<TFrontendInfo> get_frontends(); |
356 | | std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); |
357 | | |
358 | 7.74k | TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } |
359 | 948 | TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } |
360 | 0 | SchemaCache* schema_cache() { return _schema_cache; } |
361 | 43.9k | StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } |
362 | 27.9k | SegmentLoader* segment_loader() { return _segment_loader; } |
363 | 0 | LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; } |
364 | 0 | RowCache* get_row_cache() { return _row_cache; } |
365 | 1.83k | CacheManager* get_cache_manager() { return _cache_manager; } |
366 | 0 | IdManager* get_id_manager() { return _id_manager; } |
367 | 1 | ProcessProfile* get_process_profile() { return _process_profile; } |
368 | 0 | HeapProfiler* get_heap_profiler() { return _heap_profiler; } |
369 | 265 | segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() { |
370 | 265 | return _inverted_index_searcher_cache; |
371 | 265 | } |
372 | 199 | segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() { |
373 | 199 | return _inverted_index_query_cache; |
374 | 199 | } |
375 | 0 | segment_v2::ConditionCache* get_condition_cache() { return _condition_cache; } |
376 | 0 | segment_v2::EncodingInfoResolver* get_encoding_info_resolver() { |
377 | 0 | return _encoding_info_resolver; |
378 | 0 | } |
379 | 3 | QueryCache* get_query_cache() { return _query_cache; } |
380 | | |
381 | 1 | pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { |
382 | 1 | return _runtime_filter_timer_queue; |
383 | 1 | } |
384 | | |
385 | 2 | vectorized::DictionaryFactory* dict_factory() { return _dict_factory; } |
386 | | |
387 | 0 | pipeline::PipelineTracerContext* pipeline_tracer_context() { |
388 | 0 | return _pipeline_tracer_ctx.get(); |
389 | 0 | } |
390 | | |
391 | 557 | segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); } |
392 | 3.82k | io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); } |
393 | | |
394 | 52 | orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; } |
395 | 0 | arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; } |
396 | | |
397 | | bool check_auth_token(const std::string& auth_token); |
398 | 3 | void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; } |
399 | | void clear_stream_mgr(); |
400 | | |
401 | 190 | DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } |
402 | | Status init_mem_env(); |
403 | | |
404 | | private: |
405 | | ExecEnv(); |
406 | | |
407 | | [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths, |
408 | | const std::vector<StorePath>& spill_store_paths, |
409 | | const std::set<std::string>& broken_paths); |
410 | | void _destroy(); |
411 | | |
412 | | Status _check_deploy_mode(); |
413 | | |
414 | | Status _create_internal_workload_group(); |
415 | | void _init_runtime_filter_timer_queue(); |
416 | | |
417 | | inline static std::atomic_bool _s_ready {false}; |
418 | | inline static std::atomic_bool _s_tracking_memory {false}; |
419 | | std::vector<StorePath> _store_paths; |
420 | | std::vector<StorePath> _spill_store_paths; |
421 | | inline static std::atomic_bool _s_upgrading {false}; |
422 | | |
423 | | io::FileCacheFactory* _file_cache_factory = nullptr; |
424 | | UserFunctionCache* _user_function_cache = nullptr; |
425 | | // Leave protected so that subclasses can override |
426 | | ExternalScanContextMgr* _external_scan_context_mgr = nullptr; |
427 | | vectorized::VDataStreamMgr* _vstream_mgr = nullptr; |
428 | | ResultBufferMgr* _result_mgr = nullptr; |
429 | | ResultQueueMgr* _result_queue_mgr = nullptr; |
430 | | ClientCache<BackendServiceClient>* _backend_client_cache = nullptr; |
431 | | ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr; |
432 | | ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr; |
433 | | |
434 | | // The default tracker consumed by mem hook. If the thread does not attach other trackers, |
435 | | // by default all consumption will be passed to the process tracker through the orphan tracker. |
436 | | // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`. |
437 | | // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership", |
438 | | // and the consumption of the orphan mem tracker is close to 0, but greater than 0. |
439 | | std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker; |
440 | | std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker; |
441 | | // Count the memory consumption of segment compaction tasks. |
442 | | std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker; |
443 | | std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker; |
444 | | |
445 | | std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker; |
446 | | std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker; |
447 | | std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker; |
448 | | |
449 | | // Tracking memory may be shared between multiple queries. |
450 | | std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker; |
451 | | std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker; |
452 | | std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker; |
453 | | |
454 | | // TODO, looking forward to more accurate tracking. |
455 | | std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker; |
456 | | std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker; |
457 | | std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker; |
458 | | |
459 | | // Tracking memory consumption of parquet meta |
460 | | std::shared_ptr<MemTrackerLimiter> _parquet_meta_tracker; |
461 | | |
462 | | std::unique_ptr<ThreadPool> _send_batch_thread_pool; |
463 | | // Threadpool used to prefetch remote file for buffered reader |
464 | | std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool; |
465 | | // Threadpool used to send TableStats to FE |
466 | | std::unique_ptr<ThreadPool> _send_table_stats_thread_pool; |
467 | | // Threadpool used to upload local file to s3 |
468 | | std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool; |
469 | | // Pool used by join node to build hash table |
470 | | // Pool to use a new thread to release object |
471 | | std::unique_ptr<ThreadPool> _lazy_release_obj_pool; |
472 | | std::unique_ptr<ThreadPool> _non_block_close_thread_pool; |
473 | | std::unique_ptr<ThreadPool> _s3_file_system_thread_pool; |
474 | | // for java-udf to close |
475 | | std::unique_ptr<ThreadPool> _udf_close_workers_thread_pool; |
476 | | |
477 | | FragmentMgr* _fragment_mgr = nullptr; |
478 | | WorkloadGroupMgr* _workload_group_manager = nullptr; |
479 | | |
480 | | ResultCache* _result_cache = nullptr; |
481 | | ClusterInfo* _cluster_info = nullptr; |
482 | | LoadPathMgr* _load_path_mgr = nullptr; |
483 | | |
484 | | BfdParser* _bfd_parser = nullptr; |
485 | | BrokerMgr* _broker_mgr = nullptr; |
486 | | LoadChannelMgr* _load_channel_mgr = nullptr; |
487 | | std::unique_ptr<LoadStreamMgr> _load_stream_mgr; |
488 | | std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr; |
489 | | BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr; |
490 | | BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr; |
491 | | BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr; |
492 | | |
493 | | std::unique_ptr<StreamLoadExecutor> _stream_load_executor; |
494 | | RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr; |
495 | | StreamLoadRecorderManager* _stream_load_recorder_manager = nullptr; |
496 | | SmallFileMgr* _small_file_mgr = nullptr; |
497 | | HeartbeatFlags* _heartbeat_flags = nullptr; |
498 | | vectorized::ScannerScheduler* _scanner_scheduler = nullptr; |
499 | | |
500 | | // To save meta info of external file, such as parquet footer. |
501 | | FileMetaCache* _file_meta_cache = nullptr; |
502 | | std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter; |
503 | | std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool; |
504 | | std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool; |
505 | | std::unique_ptr<WalManager> _wal_manager; |
506 | | DNSCache* _dns_cache = nullptr; |
507 | | std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors; |
508 | | |
509 | | std::mutex _frontends_lock; |
510 | | // ip:brpc_port -> frontend_indo |
511 | | std::map<TNetworkAddress, FrontendInfo> _frontends; |
512 | | GroupCommitMgr* _group_commit_mgr = nullptr; |
513 | | |
514 | | // Maybe we should use unique_ptr, but it need complete type, which means we need |
515 | | // to include many headers, and for some cpp file that do not need class like TabletSchemaCache, |
516 | | // these redundancy header could introduce potential bug, at least, more header means slow compile. |
517 | | // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. |
518 | | TabletSchemaCache* _tablet_schema_cache = nullptr; |
519 | | TabletColumnObjectPool* _tablet_column_object_pool = nullptr; |
520 | | std::unique_ptr<BaseStorageEngine> _storage_engine; |
521 | | SchemaCache* _schema_cache = nullptr; |
522 | | StoragePageCache* _storage_page_cache = nullptr; |
523 | | SegmentLoader* _segment_loader = nullptr; |
524 | | LookupConnectionCache* _lookup_connection_cache = nullptr; |
525 | | RowCache* _row_cache = nullptr; |
526 | | CacheManager* _cache_manager = nullptr; |
527 | | IdManager* _id_manager = nullptr; |
528 | | ProcessProfile* _process_profile = nullptr; |
529 | | HeapProfiler* _heap_profiler = nullptr; |
530 | | segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr; |
531 | | segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr; |
532 | | segment_v2::ConditionCache* _condition_cache = nullptr; |
533 | | segment_v2::EncodingInfoResolver* _encoding_info_resolver = nullptr; |
534 | | QueryCache* _query_cache = nullptr; |
535 | | std::unique_ptr<io::FDCache> _file_cache_open_fd_cache; |
536 | | DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; |
537 | | |
538 | | pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; |
539 | | vectorized::DictionaryFactory* _dict_factory = nullptr; |
540 | | |
541 | | WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; |
542 | | IndexPolicyMgr* _index_policy_mgr = nullptr; |
543 | | |
544 | | RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr; |
545 | | |
546 | | std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx; |
547 | | std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs; |
548 | | doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr; |
549 | | |
550 | | orc::MemoryPool* _orc_memory_pool = nullptr; |
551 | | arrow::MemoryPool* _arrow_memory_pool = nullptr; |
552 | | |
553 | | kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; |
554 | | io::HdfsMgr* _hdfs_mgr = nullptr; |
555 | | }; |
556 | | |
557 | | template <> |
558 | 0 | inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() { |
559 | 0 | return _backend_client_cache; |
560 | 0 | } |
561 | | template <> |
562 | 0 | inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() { |
563 | 0 | return _frontend_client_cache; |
564 | 0 | } |
565 | | template <> |
566 | | inline ClientCache<TPaloBrokerServiceClient>* |
567 | 0 | ExecEnv::get_client_cache<TPaloBrokerServiceClient>() { |
568 | 0 | return _broker_client_cache; |
569 | 0 | } |
570 | | |
571 | 0 | inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() { |
572 | 0 | return ExecEnv::GetInstance()->get_inverted_index_query_cache(); |
573 | 0 | } |
574 | | |
575 | | } // namespace doris |