/root/doris/be/src/runtime/exec_env.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <common/multi_version.h> |
21 | | #include <gen_cpp/olap_file.pb.h> |
22 | | |
23 | | #include <atomic> |
24 | | #include <map> |
25 | | #include <memory> |
26 | | #include <mutex> |
27 | | #include <optional> |
28 | | #include <string> |
29 | | #include <vector> |
30 | | |
31 | | #include "common/config.h" |
32 | | #include "common/status.h" |
33 | | #include "exec/schema_scanner/schema_routine_load_job_scanner.h" |
34 | | #include "io/cache/fs_file_cache_storage.h" |
35 | | #include "olap/memtable_memory_limiter.h" |
36 | | #include "olap/options.h" |
37 | | #include "olap/rowset/segment_v2/inverted_index_writer.h" |
38 | | #include "olap/tablet_fwd.h" |
39 | | #include "pipeline/pipeline_tracing.h" |
40 | | #include "runtime/cluster_info.h" |
41 | | #include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header |
42 | | #include "util/threadpool.h" |
43 | | |
44 | | namespace orc { |
45 | | class MemoryPool; |
46 | | } |
47 | | namespace arrow { |
48 | | class MemoryPool; |
49 | | } |
50 | | |
51 | | namespace doris { |
52 | | namespace vectorized { |
53 | | class VDataStreamMgr; |
54 | | class SpillStreamManager; |
55 | | class DeltaWriterV2Pool; |
56 | | class DictionaryFactory; |
57 | | } // namespace vectorized |
58 | | namespace pipeline { |
59 | | class TaskScheduler; |
60 | | struct RuntimeFilterTimerQueue; |
61 | | } // namespace pipeline |
62 | | class WorkloadGroupMgr; |
63 | | struct WriteCooldownMetaExecutors; |
64 | | namespace io { |
65 | | class FileCacheFactory; |
66 | | class HdfsMgr; |
67 | | class PackedFileManager; |
68 | | } // namespace io |
69 | | namespace segment_v2 { |
70 | | class InvertedIndexSearcherCache; |
71 | | class InvertedIndexQueryCache; |
72 | | class ConditionCache; |
73 | | class TmpFileDirs; |
74 | | class EncodingInfoResolver; |
75 | | |
76 | | namespace inverted_index { |
77 | | class AnalysisFactoryMgr; |
78 | | } |
79 | | |
80 | | } // namespace segment_v2 |
81 | | |
82 | | namespace kerberos { |
83 | | class KerberosTicketMgr; |
84 | | } |
85 | | |
86 | | class QueryCache; |
87 | | class WorkloadSchedPolicyMgr; |
88 | | class BfdParser; |
89 | | class BrokerMgr; |
90 | | template <class T> |
91 | | class BrpcClientCache; |
92 | | class ExternalScanContextMgr; |
93 | | class FragmentMgr; |
94 | | class ResultCache; |
95 | | class LoadPathMgr; |
96 | | class NewLoadStreamMgr; |
97 | | class MemTrackerLimiter; |
98 | | class MemTracker; |
99 | | struct TrackerLimiterGroup; |
100 | | class BaseStorageEngine; |
101 | | class ResultBufferMgr; |
102 | | class ResultQueueMgr; |
103 | | class RuntimeQueryStatisticsMgr; |
104 | | class LoadChannelMgr; |
105 | | class LoadStreamMgr; |
106 | | class LoadStreamMapPool; |
107 | | class StreamLoadExecutor; |
108 | | class RoutineLoadTaskExecutor; |
109 | | class StreamLoadRecorderManager; |
110 | | class SmallFileMgr; |
111 | | class BackendServiceClient; |
112 | | class TPaloBrokerServiceClient; |
113 | | class PBackendService_Stub; |
114 | | class PFunctionService_Stub; |
115 | | template <class T> |
116 | | class ClientCache; |
117 | | class HeartbeatFlags; |
118 | | class FrontendServiceClient; |
119 | | class FileMetaCache; |
120 | | class GroupCommitMgr; |
121 | | class CdcClientMgr; |
122 | | class TabletSchemaCache; |
123 | | class TabletColumnObjectPool; |
124 | | class UserFunctionCache; |
125 | | class SchemaCache; |
126 | | class StoragePageCache; |
127 | | class SegmentLoader; |
128 | | class LookupConnectionCache; |
129 | | class RowCache; |
130 | | class DummyLRUCache; |
131 | | class CacheManager; |
132 | | class IdManager; |
133 | | class ProcessProfile; |
134 | | class HeapProfiler; |
135 | | class WalManager; |
136 | | class DNSCache; |
137 | | class IndexPolicyMgr; |
138 | | struct SyncRowsetStats; |
139 | | class DeleteBitmapAggCache; |
140 | | |
141 | | // set to true when BE is shutting down |
142 | | inline bool k_doris_exit = false; |
143 | | // set to true after BE start ready |
144 | | inline bool k_is_server_ready = false; |
145 | | |
146 | | // Execution environment for queries/plan fragments. |
147 | | // Contains all required global structures, and handles to |
148 | | // singleton services. Clients must call StartServices exactly |
149 | | // once to properly initialise service state. |
150 | | class ExecEnv { |
151 | | public: |
152 | | // Empty destructor because the compiler-generated one requires full |
153 | | // declarations for classes in scoped_ptrs. |
154 | | ~ExecEnv(); |
155 | | |
156 | 8.92k | BaseStorageEngine& storage_engine() { return *_storage_engine; } |
157 | | |
158 | | // Initial exec environment. must call this to init all |
159 | | [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths, |
160 | | const std::vector<StorePath>& spill_store_paths, |
161 | | const std::set<std::string>& broken_paths); |
162 | | |
163 | | // Stop all threads and delete resources. |
164 | | void destroy(); |
165 | | |
166 | | /// Returns the first created exec env instance. In a normal doris, this is |
167 | | /// the only instance. In test setups with multiple ExecEnv's per process, |
168 | | /// we return the most recently created instance. |
169 | 1.02M | static ExecEnv* GetInstance() { |
170 | 1.02M | static ExecEnv s_exec_env; |
171 | 1.02M | return &s_exec_env; |
172 | 1.02M | } |
173 | | |
174 | | // Requires ExenEnv ready |
175 | | /* |
176 | | * Parameters: |
177 | | * - tablet_id: the id of tablet to get |
178 | | * - sync_stats: the stats of sync rowset |
179 | | * - force_use_only_cached: whether only use cached data |
180 | | * - cache_on_miss: whether cache the tablet meta when missing in cache |
181 | | */ |
182 | | static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id, |
183 | | SyncRowsetStats* sync_stats = nullptr, |
184 | | bool force_use_only_cached = false, |
185 | | bool cache_on_miss = true); |
186 | | |
187 | | static Status get_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta, |
188 | | bool force_use_only_cached = false); |
189 | | |
190 | 453k | static bool ready() { return _s_ready.load(std::memory_order_acquire); } |
191 | 72 | static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); } |
192 | 11 | static bool get_is_upgrading() { return _s_upgrading.load(std::memory_order_acquire); } |
193 | 0 | static void set_is_upgrading() { _s_upgrading = true; } |
194 | | const std::string& token() const; |
195 | 0 | ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; } |
196 | 8 | vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; } |
197 | 0 | ResultBufferMgr* result_mgr() { return _result_mgr; } |
198 | 1 | ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; } |
199 | 0 | ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; } |
200 | 0 | ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; } |
201 | 0 | ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; } |
202 | | |
203 | 1.79k | WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; } |
204 | 0 | WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; } |
205 | 0 | RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() { |
206 | 0 | return _runtime_query_statistics_mgr; |
207 | 0 | } |
208 | | |
209 | | // using template to simplify client cache management |
210 | | template <typename T> |
211 | | inline ClientCache<T>* get_client_cache() { |
212 | | return nullptr; |
213 | | } |
214 | | |
215 | | // Save all MemTrackerLimiters in use. |
216 | | // Each group corresponds to several MemTrackerLimiters and has a lock. |
217 | | // Multiple groups are used to reduce the impact of locks. |
218 | | std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool; |
219 | | void init_mem_tracker(); |
220 | 447k | std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; } |
221 | 0 | std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() { |
222 | 0 | return _brpc_iobuf_block_memory_tracker; |
223 | 0 | } |
224 | 0 | std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() { |
225 | 0 | return _segcompaction_mem_tracker; |
226 | 0 | } |
227 | 8 | std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() { |
228 | 8 | return _stream_load_pipe_tracker; |
229 | 8 | } |
230 | 0 | std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() { |
231 | 0 | return _tablets_no_cache_mem_tracker; |
232 | 0 | } |
233 | 0 | std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() { |
234 | 0 | return _rowsets_no_cache_mem_tracker; |
235 | 0 | } |
236 | 0 | std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() { |
237 | 0 | return _segments_no_cache_mem_tracker; |
238 | 0 | } |
239 | 2.09k | std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() { |
240 | 2.09k | return _point_query_executor_mem_tracker; |
241 | 2.09k | } |
242 | 12 | std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() { |
243 | 12 | return _query_cache_mem_tracker; |
244 | 12 | } |
245 | 26.2k | std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() { |
246 | 26.2k | return _block_compression_mem_tracker; |
247 | 26.2k | } |
248 | 0 | std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() { |
249 | 0 | return _rowid_storage_reader_tracker; |
250 | 0 | } |
251 | 2.11k | std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() { |
252 | 2.11k | return _subcolumns_tree_tracker; |
253 | 2.11k | } |
254 | 2.57k | std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; } |
255 | 216 | std::shared_ptr<MemTrackerLimiter> parquet_meta_tracker() { return _parquet_meta_tracker; } |
256 | | |
257 | 0 | ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); } |
258 | 4 | ThreadPool* buffered_reader_prefetch_thread_pool() { |
259 | 4 | return _buffered_reader_prefetch_thread_pool.get(); |
260 | 4 | } |
261 | 0 | ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); } |
262 | 1.28k | ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); } |
263 | 0 | ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); } |
264 | | ThreadPool* non_block_close_thread_pool(); |
265 | 0 | ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); } |
266 | 0 | ThreadPool* udf_close_workers_pool() { return _udf_close_workers_thread_pool.get(); } |
267 | | |
268 | | void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths); |
269 | 2.17k | io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; } |
270 | 0 | UserFunctionCache* user_function_cache() { return _user_function_cache; } |
271 | 121k | FragmentMgr* fragment_mgr() { return _fragment_mgr; } |
272 | 0 | ResultCache* result_cache() { return _result_cache; } |
273 | 9 | ClusterInfo* cluster_info() { return _cluster_info; } |
274 | 1 | LoadPathMgr* load_path_mgr() { return _load_path_mgr; } |
275 | 1 | BfdParser* bfd_parser() const { return _bfd_parser; } |
276 | 0 | BrokerMgr* broker_mgr() const { return _broker_mgr; } |
277 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const { |
278 | 0 | return _internal_client_cache; |
279 | 0 | } |
280 | 0 | BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const { |
281 | 0 | return _streaming_client_cache; |
282 | 0 | } |
283 | 0 | BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const { |
284 | 0 | return _function_client_cache; |
285 | 0 | } |
286 | 0 | LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; } |
287 | 0 | LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); } |
288 | 3 | NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); } |
289 | 0 | SmallFileMgr* small_file_mgr() { return _small_file_mgr; } |
290 | 612 | doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; } |
291 | 0 | GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; } |
292 | 0 | CdcClientMgr* cdc_client_mgr() { return _cdc_client_mgr; } |
293 | | |
294 | 3 | const std::vector<StorePath>& store_paths() const { return _store_paths; } |
295 | | |
296 | 1 | StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); } |
297 | 0 | RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; } |
298 | 0 | HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; } |
299 | 14 | FileMetaCache* file_meta_cache() { return _file_meta_cache; } |
300 | 16 | MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); } |
301 | 44 | WalManager* wal_mgr() { return _wal_manager.get(); } |
302 | 8 | DNSCache* dns_cache() { return _dns_cache; } |
303 | 5 | WriteCooldownMetaExecutors* write_cooldown_meta_executors() { |
304 | 5 | return _write_cooldown_meta_executors.get(); |
305 | 5 | } |
306 | | |
307 | 0 | kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } |
308 | 0 | io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } |
309 | 0 | io::PackedFileManager* packed_file_manager() { return _packed_file_manager; } |
310 | 3 | IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; } |
311 | | |
312 | | #ifdef BE_TEST |
313 | 217 | void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) { |
314 | 217 | this->_tmp_file_dirs = std::move(tmp_file_dirs); |
315 | 217 | } |
316 | 4 | void set_ready() { this->_s_ready = true; } |
317 | 0 | void set_not_ready() { this->_s_ready = false; } |
318 | 36 | void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { |
319 | 36 | _memtable_memory_limiter.reset(limiter); |
320 | 36 | } |
321 | 3 | void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } |
322 | | void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr); |
323 | | void clear_new_load_stream_mgr(); |
324 | | void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor); |
325 | | void clear_stream_load_executor(); |
326 | | |
327 | | void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine); |
328 | | void set_inverted_index_searcher_cache( |
329 | | segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); |
330 | 1 | void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } |
331 | 1 | void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } |
332 | 1 | void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } |
333 | 1 | void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } |
334 | 1 | void set_tablet_column_object_pool(TabletColumnObjectPool* c) { |
335 | 1 | this->_tablet_column_object_pool = c; |
336 | 1 | } |
337 | 1 | void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } |
338 | 1 | void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } |
339 | 0 | void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { |
340 | 0 | this->_routine_load_task_executor = r; |
341 | 0 | } |
342 | | void set_wal_mgr(std::unique_ptr<WalManager>&& wm); |
343 | | void clear_wal_mgr(); |
344 | | |
345 | | void set_write_cooldown_meta_executors(); |
346 | 1 | static void set_tracking_memory(bool tracking_memory) { |
347 | 1 | _s_tracking_memory.store(tracking_memory, std::memory_order_release); |
348 | 1 | } |
349 | 1 | void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } |
350 | 2 | void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
351 | 2 | _non_block_close_thread_pool = std::move(pool); |
352 | 2 | } |
353 | 2 | void set_s3_file_upload_thread_pool(std::unique_ptr<ThreadPool>&& pool) { |
354 | 2 | _s3_file_upload_thread_pool = std::move(pool); |
355 | 2 | } |
356 | 0 | void set_file_cache_factory(io::FileCacheFactory* factory) { _file_cache_factory = factory; } |
357 | 2 | void set_file_cache_open_fd_cache(std::unique_ptr<io::FDCache>&& fd_cache) { |
358 | 2 | _file_cache_open_fd_cache = std::move(fd_cache); |
359 | 2 | } |
360 | | #endif |
361 | 0 | LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); } |
362 | | |
363 | 0 | vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); } |
364 | | |
365 | | void wait_for_all_tasks_done(); |
366 | | |
367 | | void update_frontends(const std::vector<TFrontendInfo>& new_infos); |
368 | | std::vector<TFrontendInfo> get_frontends(); |
369 | | std::map<TNetworkAddress, FrontendInfo> get_running_frontends(); |
370 | | |
371 | 7.84k | TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } |
372 | 1.01k | TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } |
373 | 0 | SchemaCache* schema_cache() { return _schema_cache; } |
374 | 44.0k | StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } |
375 | 27.9k | SegmentLoader* segment_loader() { return _segment_loader; } |
376 | 0 | LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; } |
377 | 0 | RowCache* get_row_cache() { return _row_cache; } |
378 | 1.93k | CacheManager* get_cache_manager() { return _cache_manager; } |
379 | 0 | IdManager* get_id_manager() { return _id_manager; } |
380 | 1 | ProcessProfile* get_process_profile() { return _process_profile; } |
381 | 0 | HeapProfiler* get_heap_profiler() { return _heap_profiler; } |
382 | 277 | segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() { |
383 | 277 | return _inverted_index_searcher_cache; |
384 | 277 | } |
385 | 221 | segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() { |
386 | 221 | return _inverted_index_query_cache; |
387 | 221 | } |
388 | 0 | segment_v2::ConditionCache* get_condition_cache() { return _condition_cache; } |
389 | 0 | segment_v2::EncodingInfoResolver* get_encoding_info_resolver() { |
390 | 0 | return _encoding_info_resolver; |
391 | 0 | } |
392 | 3 | QueryCache* get_query_cache() { return _query_cache; } |
393 | | |
394 | 1 | pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { |
395 | 1 | return _runtime_filter_timer_queue; |
396 | 1 | } |
397 | | |
398 | 2 | vectorized::DictionaryFactory* dict_factory() { return _dict_factory; } |
399 | | |
400 | 0 | pipeline::PipelineTracerContext* pipeline_tracer_context() { |
401 | 0 | return _pipeline_tracer_ctx.get(); |
402 | 0 | } |
403 | | |
404 | 574 | segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); } |
405 | 16.1k | io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); } |
406 | | |
407 | 52 | orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; } |
408 | 0 | arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; } |
409 | | |
410 | | bool check_auth_token(const std::string& auth_token); |
411 | 3 | void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; } |
412 | | void clear_stream_mgr(); |
413 | | |
414 | 190 | DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } |
415 | | Status init_mem_env(); |
416 | | |
417 | | private: |
418 | | ExecEnv(); |
419 | | |
420 | | [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths, |
421 | | const std::vector<StorePath>& spill_store_paths, |
422 | | const std::set<std::string>& broken_paths); |
423 | | void _destroy(); |
424 | | |
425 | | Status _check_deploy_mode(); |
426 | | |
427 | | Status _create_internal_workload_group(); |
428 | | void _init_runtime_filter_timer_queue(); |
429 | | |
430 | | inline static std::atomic_bool _s_ready {false}; |
431 | | inline static std::atomic_bool _s_tracking_memory {false}; |
432 | | std::vector<StorePath> _store_paths; |
433 | | std::vector<StorePath> _spill_store_paths; |
434 | | inline static std::atomic_bool _s_upgrading {false}; |
435 | | |
436 | | io::FileCacheFactory* _file_cache_factory = nullptr; |
437 | | UserFunctionCache* _user_function_cache = nullptr; |
438 | | // Leave protected so that subclasses can override |
439 | | ExternalScanContextMgr* _external_scan_context_mgr = nullptr; |
440 | | vectorized::VDataStreamMgr* _vstream_mgr = nullptr; |
441 | | ResultBufferMgr* _result_mgr = nullptr; |
442 | | ResultQueueMgr* _result_queue_mgr = nullptr; |
443 | | ClientCache<BackendServiceClient>* _backend_client_cache = nullptr; |
444 | | ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr; |
445 | | ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr; |
446 | | |
447 | | // The default tracker consumed by mem hook. If the thread does not attach other trackers, |
448 | | // by default all consumption will be passed to the process tracker through the orphan tracker. |
449 | | // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`. |
450 | | // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership", |
451 | | // and the consumption of the orphan mem tracker is close to 0, but greater than 0. |
452 | | std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker; |
453 | | std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker; |
454 | | // Count the memory consumption of segment compaction tasks. |
455 | | std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker; |
456 | | std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker; |
457 | | |
458 | | std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker; |
459 | | std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker; |
460 | | std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker; |
461 | | |
462 | | // Tracking memory may be shared between multiple queries. |
463 | | std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker; |
464 | | std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker; |
465 | | std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker; |
466 | | |
467 | | // TODO, looking forward to more accurate tracking. |
468 | | std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker; |
469 | | std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker; |
470 | | std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker; |
471 | | |
472 | | // Tracking memory consumption of parquet meta |
473 | | std::shared_ptr<MemTrackerLimiter> _parquet_meta_tracker; |
474 | | |
475 | | std::unique_ptr<ThreadPool> _send_batch_thread_pool; |
476 | | // Threadpool used to prefetch remote file for buffered reader |
477 | | std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool; |
478 | | // Threadpool used to send TableStats to FE |
479 | | std::unique_ptr<ThreadPool> _send_table_stats_thread_pool; |
480 | | // Threadpool used to upload local file to s3 |
481 | | std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool; |
482 | | // Pool used by join node to build hash table |
483 | | // Pool to use a new thread to release object |
484 | | std::unique_ptr<ThreadPool> _lazy_release_obj_pool; |
485 | | std::unique_ptr<ThreadPool> _non_block_close_thread_pool; |
486 | | std::unique_ptr<ThreadPool> _s3_file_system_thread_pool; |
487 | | // for java-udf to close |
488 | | std::unique_ptr<ThreadPool> _udf_close_workers_thread_pool; |
489 | | |
490 | | FragmentMgr* _fragment_mgr = nullptr; |
491 | | WorkloadGroupMgr* _workload_group_manager = nullptr; |
492 | | |
493 | | ResultCache* _result_cache = nullptr; |
494 | | ClusterInfo* _cluster_info = nullptr; |
495 | | LoadPathMgr* _load_path_mgr = nullptr; |
496 | | |
497 | | BfdParser* _bfd_parser = nullptr; |
498 | | BrokerMgr* _broker_mgr = nullptr; |
499 | | LoadChannelMgr* _load_channel_mgr = nullptr; |
500 | | std::unique_ptr<LoadStreamMgr> _load_stream_mgr; |
501 | | std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr; |
502 | | BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr; |
503 | | BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr; |
504 | | BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr; |
505 | | |
506 | | std::unique_ptr<StreamLoadExecutor> _stream_load_executor; |
507 | | RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr; |
508 | | StreamLoadRecorderManager* _stream_load_recorder_manager = nullptr; |
509 | | SmallFileMgr* _small_file_mgr = nullptr; |
510 | | HeartbeatFlags* _heartbeat_flags = nullptr; |
511 | | |
512 | | // To save meta info of external file, such as parquet footer. |
513 | | FileMetaCache* _file_meta_cache = nullptr; |
514 | | std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter; |
515 | | std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool; |
516 | | std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool; |
517 | | std::unique_ptr<WalManager> _wal_manager; |
518 | | DNSCache* _dns_cache = nullptr; |
519 | | std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors; |
520 | | |
521 | | std::mutex _frontends_lock; |
522 | | // ip:brpc_port -> frontend_indo |
523 | | std::map<TNetworkAddress, FrontendInfo> _frontends; |
524 | | GroupCommitMgr* _group_commit_mgr = nullptr; |
525 | | CdcClientMgr* _cdc_client_mgr = nullptr; |
526 | | |
527 | | // Maybe we should use unique_ptr, but it need complete type, which means we need |
528 | | // to include many headers, and for some cpp file that do not need class like TabletSchemaCache, |
529 | | // these redundancy header could introduce potential bug, at least, more header means slow compile. |
530 | | // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. |
531 | | TabletSchemaCache* _tablet_schema_cache = nullptr; |
532 | | TabletColumnObjectPool* _tablet_column_object_pool = nullptr; |
533 | | std::unique_ptr<BaseStorageEngine> _storage_engine; |
534 | | SchemaCache* _schema_cache = nullptr; |
535 | | StoragePageCache* _storage_page_cache = nullptr; |
536 | | SegmentLoader* _segment_loader = nullptr; |
537 | | LookupConnectionCache* _lookup_connection_cache = nullptr; |
538 | | RowCache* _row_cache = nullptr; |
539 | | CacheManager* _cache_manager = nullptr; |
540 | | IdManager* _id_manager = nullptr; |
541 | | ProcessProfile* _process_profile = nullptr; |
542 | | HeapProfiler* _heap_profiler = nullptr; |
543 | | segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr; |
544 | | segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr; |
545 | | segment_v2::ConditionCache* _condition_cache = nullptr; |
546 | | segment_v2::EncodingInfoResolver* _encoding_info_resolver = nullptr; |
547 | | QueryCache* _query_cache = nullptr; |
548 | | std::unique_ptr<io::FDCache> _file_cache_open_fd_cache; |
549 | | DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; |
550 | | |
551 | | pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; |
552 | | vectorized::DictionaryFactory* _dict_factory = nullptr; |
553 | | |
554 | | WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; |
555 | | IndexPolicyMgr* _index_policy_mgr = nullptr; |
556 | | |
557 | | RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr; |
558 | | |
559 | | std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx; |
560 | | std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs; |
561 | | doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr; |
562 | | |
563 | | orc::MemoryPool* _orc_memory_pool = nullptr; |
564 | | arrow::MemoryPool* _arrow_memory_pool = nullptr; |
565 | | |
566 | | kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; |
567 | | io::HdfsMgr* _hdfs_mgr = nullptr; |
568 | | io::PackedFileManager* _packed_file_manager = nullptr; |
569 | | }; |
570 | | |
571 | | template <> |
572 | 0 | inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() { |
573 | 0 | return _backend_client_cache; |
574 | 0 | } |
575 | | template <> |
576 | 0 | inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() { |
577 | 0 | return _frontend_client_cache; |
578 | 0 | } |
579 | | template <> |
580 | | inline ClientCache<TPaloBrokerServiceClient>* |
581 | 0 | ExecEnv::get_client_cache<TPaloBrokerServiceClient>() { |
582 | 0 | return _broker_client_cache; |
583 | 0 | } |
584 | | |
585 | 0 | inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() { |
586 | 0 | return ExecEnv::GetInstance()->get_inverted_index_query_cache(); |
587 | 0 | } |
588 | | |
589 | | } // namespace doris |