Coverage Report

Created: 2025-12-26 15:36

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/runtime/exec_env.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <common/multi_version.h>
21
#include <gen_cpp/olap_file.pb.h>
22
23
#include <atomic>
24
#include <map>
25
#include <memory>
26
#include <mutex>
27
#include <optional>
28
#include <string>
29
#include <vector>
30
31
#include "common/config.h"
32
#include "common/status.h"
33
#include "exec/schema_scanner/schema_routine_load_job_scanner.h"
34
#include "io/cache/fs_file_cache_storage.h"
35
#include "olap/memtable_memory_limiter.h"
36
#include "olap/options.h"
37
#include "olap/rowset/segment_v2/inverted_index_writer.h"
38
#include "olap/tablet_fwd.h"
39
#include "pipeline/pipeline_tracing.h"
40
#include "runtime/cluster_info.h"
41
#include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header
42
#include "util/threadpool.h"
43
44
namespace orc {
45
class MemoryPool;
46
}
47
namespace arrow {
48
class MemoryPool;
49
}
50
51
namespace doris {
52
namespace vectorized {
53
class VDataStreamMgr;
54
class SpillStreamManager;
55
class DeltaWriterV2Pool;
56
class DictionaryFactory;
57
} // namespace vectorized
58
namespace pipeline {
59
class TaskScheduler;
60
struct RuntimeFilterTimerQueue;
61
} // namespace pipeline
62
class WorkloadGroupMgr;
63
struct WriteCooldownMetaExecutors;
64
namespace io {
65
class FileCacheFactory;
66
class HdfsMgr;
67
class PackedFileManager;
68
} // namespace io
69
namespace segment_v2 {
70
class InvertedIndexSearcherCache;
71
class InvertedIndexQueryCache;
72
class ConditionCache;
73
class TmpFileDirs;
74
class EncodingInfoResolver;
75
76
namespace inverted_index {
77
class AnalysisFactoryMgr;
78
}
79
80
} // namespace segment_v2
81
82
namespace kerberos {
83
class KerberosTicketMgr;
84
}
85
86
class QueryCache;
87
class WorkloadSchedPolicyMgr;
88
class BfdParser;
89
class BrokerMgr;
90
template <class T>
91
class BrpcClientCache;
92
class ExternalScanContextMgr;
93
class FragmentMgr;
94
class ResultCache;
95
class LoadPathMgr;
96
class NewLoadStreamMgr;
97
class MemTrackerLimiter;
98
class MemTracker;
99
struct TrackerLimiterGroup;
100
class BaseStorageEngine;
101
class ResultBufferMgr;
102
class ResultQueueMgr;
103
class RuntimeQueryStatisticsMgr;
104
class LoadChannelMgr;
105
class LoadStreamMgr;
106
class LoadStreamMapPool;
107
class StreamLoadExecutor;
108
class RoutineLoadTaskExecutor;
109
class StreamLoadRecorderManager;
110
class SmallFileMgr;
111
class BackendServiceClient;
112
class TPaloBrokerServiceClient;
113
class PBackendService_Stub;
114
class PFunctionService_Stub;
115
template <class T>
116
class ClientCache;
117
class HeartbeatFlags;
118
class FrontendServiceClient;
119
class FileMetaCache;
120
class GroupCommitMgr;
121
class CdcClientMgr;
122
class TabletSchemaCache;
123
class TabletColumnObjectPool;
124
class UserFunctionCache;
125
class SchemaCache;
126
class StoragePageCache;
127
class SegmentLoader;
128
class LookupConnectionCache;
129
class RowCache;
130
class DummyLRUCache;
131
class CacheManager;
132
class IdManager;
133
class ProcessProfile;
134
class HeapProfiler;
135
class WalManager;
136
class DNSCache;
137
class IndexPolicyMgr;
138
struct SyncRowsetStats;
139
class DeleteBitmapAggCache;
140
141
// set to true when BE is shutting down
142
inline bool k_doris_exit = false;
143
// set to true after BE start ready
144
inline bool k_is_server_ready = false;
145
146
// Execution environment for queries/plan fragments.
147
// Contains all required global structures, and handles to
148
// singleton services. Clients must call StartServices exactly
149
// once to properly initialise service state.
150
class ExecEnv {
151
public:
152
    // Empty destructor because the compiler-generated one requires full
153
    // declarations for classes in scoped_ptrs.
154
    ~ExecEnv();
155
156
8.92k
    BaseStorageEngine& storage_engine() { return *_storage_engine; }
157
158
    // Initial exec environment. must call this to init all
159
    [[nodiscard]] static Status init(ExecEnv* env, const std::vector<StorePath>& store_paths,
160
                                     const std::vector<StorePath>& spill_store_paths,
161
                                     const std::set<std::string>& broken_paths);
162
163
    // Stop all threads and delete resources.
164
    void destroy();
165
166
    /// Returns the first created exec env instance. In a normal doris, this is
167
    /// the only instance. In test setups with multiple ExecEnv's per process,
168
    /// we return the most recently created instance.
169
1.02M
    static ExecEnv* GetInstance() {
170
1.02M
        static ExecEnv s_exec_env;
171
1.02M
        return &s_exec_env;
172
1.02M
    }
173
174
    // Requires ExenEnv ready
175
    /*
176
     * Parameters:
177
     * - tablet_id: the id of tablet to get
178
     * - sync_stats: the stats of sync rowset
179
     * - force_use_only_cached: whether only use cached data
180
     * - cache_on_miss: whether cache the tablet meta when missing in cache
181
     */
182
    static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id,
183
                                             SyncRowsetStats* sync_stats = nullptr,
184
                                             bool force_use_only_cached = false,
185
                                             bool cache_on_miss = true);
186
187
    static Status get_tablet_meta(int64_t tablet_id, TabletMetaSharedPtr* tablet_meta,
188
                                  bool force_use_only_cached = false);
189
190
453k
    static bool ready() { return _s_ready.load(std::memory_order_acquire); }
191
72
    static bool tracking_memory() { return _s_tracking_memory.load(std::memory_order_acquire); }
192
11
    static bool get_is_upgrading() { return _s_upgrading.load(std::memory_order_acquire); }
193
0
    static void set_is_upgrading() { _s_upgrading = true; }
194
    const std::string& token() const;
195
0
    ExternalScanContextMgr* external_scan_context_mgr() { return _external_scan_context_mgr; }
196
8
    vectorized::VDataStreamMgr* vstream_mgr() { return _vstream_mgr; }
197
0
    ResultBufferMgr* result_mgr() { return _result_mgr; }
198
1
    ResultQueueMgr* result_queue_mgr() { return _result_queue_mgr; }
199
0
    ClientCache<BackendServiceClient>* client_cache() { return _backend_client_cache; }
200
0
    ClientCache<FrontendServiceClient>* frontend_client_cache() { return _frontend_client_cache; }
201
0
    ClientCache<TPaloBrokerServiceClient>* broker_client_cache() { return _broker_client_cache; }
202
203
1.79k
    WorkloadGroupMgr* workload_group_mgr() { return _workload_group_manager; }
204
0
    WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; }
205
0
    RuntimeQueryStatisticsMgr* runtime_query_statistics_mgr() {
206
0
        return _runtime_query_statistics_mgr;
207
0
    }
208
209
    // using template to simplify client cache management
210
    template <typename T>
211
    inline ClientCache<T>* get_client_cache() {
212
        return nullptr;
213
    }
214
215
    // Save all MemTrackerLimiters in use.
216
    // Each group corresponds to several MemTrackerLimiters and has a lock.
217
    // Multiple groups are used to reduce the impact of locks.
218
    std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool;
219
    void init_mem_tracker();
220
447k
    std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; }
221
0
    std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() {
222
0
        return _brpc_iobuf_block_memory_tracker;
223
0
    }
224
0
    std::shared_ptr<MemTrackerLimiter> segcompaction_mem_tracker() {
225
0
        return _segcompaction_mem_tracker;
226
0
    }
227
8
    std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() {
228
8
        return _stream_load_pipe_tracker;
229
8
    }
230
0
    std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() {
231
0
        return _tablets_no_cache_mem_tracker;
232
0
    }
233
0
    std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() {
234
0
        return _rowsets_no_cache_mem_tracker;
235
0
    }
236
0
    std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() {
237
0
        return _segments_no_cache_mem_tracker;
238
0
    }
239
2.09k
    std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() {
240
2.09k
        return _point_query_executor_mem_tracker;
241
2.09k
    }
242
12
    std::shared_ptr<MemTrackerLimiter> query_cache_mem_tracker() {
243
12
        return _query_cache_mem_tracker;
244
12
    }
245
26.2k
    std::shared_ptr<MemTrackerLimiter> block_compression_mem_tracker() {
246
26.2k
        return _block_compression_mem_tracker;
247
26.2k
    }
248
0
    std::shared_ptr<MemTrackerLimiter> rowid_storage_reader_tracker() {
249
0
        return _rowid_storage_reader_tracker;
250
0
    }
251
2.11k
    std::shared_ptr<MemTrackerLimiter> subcolumns_tree_tracker() {
252
2.11k
        return _subcolumns_tree_tracker;
253
2.11k
    }
254
2.57k
    std::shared_ptr<MemTrackerLimiter> s3_file_buffer_tracker() { return _s3_file_buffer_tracker; }
255
216
    std::shared_ptr<MemTrackerLimiter> parquet_meta_tracker() { return _parquet_meta_tracker; }
256
257
0
    ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); }
258
4
    ThreadPool* buffered_reader_prefetch_thread_pool() {
259
4
        return _buffered_reader_prefetch_thread_pool.get();
260
4
    }
261
0
    ThreadPool* send_table_stats_thread_pool() { return _send_table_stats_thread_pool.get(); }
262
1.28k
    ThreadPool* s3_file_upload_thread_pool() { return _s3_file_upload_thread_pool.get(); }
263
0
    ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get(); }
264
    ThreadPool* non_block_close_thread_pool();
265
0
    ThreadPool* s3_file_system_thread_pool() { return _s3_file_system_thread_pool.get(); }
266
0
    ThreadPool* udf_close_workers_pool() { return _udf_close_workers_thread_pool.get(); }
267
268
    void init_file_cache_factory(std::vector<doris::CachePath>& cache_paths);
269
2.17k
    io::FileCacheFactory* file_cache_factory() { return _file_cache_factory; }
270
0
    UserFunctionCache* user_function_cache() { return _user_function_cache; }
271
121k
    FragmentMgr* fragment_mgr() { return _fragment_mgr; }
272
0
    ResultCache* result_cache() { return _result_cache; }
273
9
    ClusterInfo* cluster_info() { return _cluster_info; }
274
1
    LoadPathMgr* load_path_mgr() { return _load_path_mgr; }
275
1
    BfdParser* bfd_parser() const { return _bfd_parser; }
276
0
    BrokerMgr* broker_mgr() const { return _broker_mgr; }
277
0
    BrpcClientCache<PBackendService_Stub>* brpc_internal_client_cache() const {
278
0
        return _internal_client_cache;
279
0
    }
280
0
    BrpcClientCache<PBackendService_Stub>* brpc_streaming_client_cache() const {
281
0
        return _streaming_client_cache;
282
0
    }
283
0
    BrpcClientCache<PFunctionService_Stub>* brpc_function_client_cache() const {
284
0
        return _function_client_cache;
285
0
    }
286
0
    LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; }
287
0
    LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr.get(); }
288
3
    NewLoadStreamMgr* new_load_stream_mgr() { return _new_load_stream_mgr.get(); }
289
0
    SmallFileMgr* small_file_mgr() { return _small_file_mgr; }
290
612
    doris::vectorized::SpillStreamManager* spill_stream_mgr() { return _spill_stream_mgr; }
291
0
    GroupCommitMgr* group_commit_mgr() { return _group_commit_mgr; }
292
0
    CdcClientMgr* cdc_client_mgr() { return _cdc_client_mgr; }
293
294
3
    const std::vector<StorePath>& store_paths() const { return _store_paths; }
295
296
1
    StreamLoadExecutor* stream_load_executor() { return _stream_load_executor.get(); }
297
0
    RoutineLoadTaskExecutor* routine_load_task_executor() { return _routine_load_task_executor; }
298
0
    HeartbeatFlags* heartbeat_flags() { return _heartbeat_flags; }
299
14
    FileMetaCache* file_meta_cache() { return _file_meta_cache; }
300
16
    MemTableMemoryLimiter* memtable_memory_limiter() { return _memtable_memory_limiter.get(); }
301
44
    WalManager* wal_mgr() { return _wal_manager.get(); }
302
8
    DNSCache* dns_cache() { return _dns_cache; }
303
5
    WriteCooldownMetaExecutors* write_cooldown_meta_executors() {
304
5
        return _write_cooldown_meta_executors.get();
305
5
    }
306
307
0
    kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; }
308
0
    io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; }
309
0
    io::PackedFileManager* packed_file_manager() { return _packed_file_manager; }
310
3
    IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; }
311
312
#ifdef BE_TEST
313
217
    void set_tmp_file_dir(std::unique_ptr<segment_v2::TmpFileDirs> tmp_file_dirs) {
314
217
        this->_tmp_file_dirs = std::move(tmp_file_dirs);
315
217
    }
316
4
    void set_ready() { this->_s_ready = true; }
317
0
    void set_not_ready() { this->_s_ready = false; }
318
36
    void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) {
319
36
        _memtable_memory_limiter.reset(limiter);
320
36
    }
321
3
    void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; }
322
    void set_new_load_stream_mgr(std::unique_ptr<NewLoadStreamMgr>&& new_load_stream_mgr);
323
    void clear_new_load_stream_mgr();
324
    void set_stream_load_executor(std::unique_ptr<StreamLoadExecutor>&& stream_load_executor);
325
    void clear_stream_load_executor();
326
327
    void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine);
328
    void set_inverted_index_searcher_cache(
329
            segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache);
330
1
    void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; }
331
1
    void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; }
332
1
    void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; }
333
1
    void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; }
334
1
    void set_tablet_column_object_pool(TabletColumnObjectPool* c) {
335
1
        this->_tablet_column_object_pool = c;
336
1
    }
337
1
    void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; }
338
1
    void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; }
339
0
    void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) {
340
0
        this->_routine_load_task_executor = r;
341
0
    }
342
    void set_wal_mgr(std::unique_ptr<WalManager>&& wm);
343
    void clear_wal_mgr();
344
345
    void set_write_cooldown_meta_executors();
346
1
    static void set_tracking_memory(bool tracking_memory) {
347
1
        _s_tracking_memory.store(tracking_memory, std::memory_order_release);
348
1
    }
349
1
    void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; }
350
2
    void set_non_block_close_thread_pool(std::unique_ptr<ThreadPool>&& pool) {
351
2
        _non_block_close_thread_pool = std::move(pool);
352
2
    }
353
2
    void set_s3_file_upload_thread_pool(std::unique_ptr<ThreadPool>&& pool) {
354
2
        _s3_file_upload_thread_pool = std::move(pool);
355
2
    }
356
0
    void set_file_cache_factory(io::FileCacheFactory* factory) { _file_cache_factory = factory; }
357
2
    void set_file_cache_open_fd_cache(std::unique_ptr<io::FDCache>&& fd_cache) {
358
2
        _file_cache_open_fd_cache = std::move(fd_cache);
359
2
    }
360
#endif
361
0
    LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); }
362
363
0
    vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); }
364
365
    void wait_for_all_tasks_done();
366
367
    void update_frontends(const std::vector<TFrontendInfo>& new_infos);
368
    std::vector<TFrontendInfo> get_frontends();
369
    std::map<TNetworkAddress, FrontendInfo> get_running_frontends();
370
371
7.84k
    TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; }
372
1.01k
    TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; }
373
0
    SchemaCache* schema_cache() { return _schema_cache; }
374
44.0k
    StoragePageCache* get_storage_page_cache() { return _storage_page_cache; }
375
27.9k
    SegmentLoader* segment_loader() { return _segment_loader; }
376
0
    LookupConnectionCache* get_lookup_connection_cache() { return _lookup_connection_cache; }
377
0
    RowCache* get_row_cache() { return _row_cache; }
378
1.93k
    CacheManager* get_cache_manager() { return _cache_manager; }
379
0
    IdManager* get_id_manager() { return _id_manager; }
380
1
    ProcessProfile* get_process_profile() { return _process_profile; }
381
0
    HeapProfiler* get_heap_profiler() { return _heap_profiler; }
382
277
    segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() {
383
277
        return _inverted_index_searcher_cache;
384
277
    }
385
221
    segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() {
386
221
        return _inverted_index_query_cache;
387
221
    }
388
0
    segment_v2::ConditionCache* get_condition_cache() { return _condition_cache; }
389
0
    segment_v2::EncodingInfoResolver* get_encoding_info_resolver() {
390
0
        return _encoding_info_resolver;
391
0
    }
392
3
    QueryCache* get_query_cache() { return _query_cache; }
393
394
1
    pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() {
395
1
        return _runtime_filter_timer_queue;
396
1
    }
397
398
2
    vectorized::DictionaryFactory* dict_factory() { return _dict_factory; }
399
400
0
    pipeline::PipelineTracerContext* pipeline_tracer_context() {
401
0
        return _pipeline_tracer_ctx.get();
402
0
    }
403
404
574
    segment_v2::TmpFileDirs* get_tmp_file_dirs() { return _tmp_file_dirs.get(); }
405
16.1k
    io::FDCache* file_cache_open_fd_cache() const { return _file_cache_open_fd_cache.get(); }
406
407
52
    orc::MemoryPool* orc_memory_pool() { return _orc_memory_pool; }
408
0
    arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; }
409
410
    bool check_auth_token(const std::string& auth_token);
411
3
    void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; }
412
    void clear_stream_mgr();
413
414
190
    DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; }
415
    Status init_mem_env();
416
417
private:
418
    ExecEnv();
419
420
    [[nodiscard]] Status _init(const std::vector<StorePath>& store_paths,
421
                               const std::vector<StorePath>& spill_store_paths,
422
                               const std::set<std::string>& broken_paths);
423
    void _destroy();
424
425
    Status _check_deploy_mode();
426
427
    Status _create_internal_workload_group();
428
    void _init_runtime_filter_timer_queue();
429
430
    inline static std::atomic_bool _s_ready {false};
431
    inline static std::atomic_bool _s_tracking_memory {false};
432
    std::vector<StorePath> _store_paths;
433
    std::vector<StorePath> _spill_store_paths;
434
    inline static std::atomic_bool _s_upgrading {false};
435
436
    io::FileCacheFactory* _file_cache_factory = nullptr;
437
    UserFunctionCache* _user_function_cache = nullptr;
438
    // Leave protected so that subclasses can override
439
    ExternalScanContextMgr* _external_scan_context_mgr = nullptr;
440
    vectorized::VDataStreamMgr* _vstream_mgr = nullptr;
441
    ResultBufferMgr* _result_mgr = nullptr;
442
    ResultQueueMgr* _result_queue_mgr = nullptr;
443
    ClientCache<BackendServiceClient>* _backend_client_cache = nullptr;
444
    ClientCache<FrontendServiceClient>* _frontend_client_cache = nullptr;
445
    ClientCache<TPaloBrokerServiceClient>* _broker_client_cache = nullptr;
446
447
    // The default tracker consumed by mem hook. If the thread does not attach other trackers,
448
    // by default all consumption will be passed to the process tracker through the orphan tracker.
449
    // In real time, `consumption of all limiter trackers` + `orphan tracker consumption` = `process tracker consumption`.
450
    // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership",
451
    // and the consumption of the orphan mem tracker is close to 0, but greater than 0.
452
    std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker;
453
    std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker;
454
    // Count the memory consumption of segment compaction tasks.
455
    std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker;
456
    std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker;
457
458
    std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker;
459
    std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker;
460
    std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker;
461
462
    // Tracking memory may be shared between multiple queries.
463
    std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker;
464
    std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker;
465
    std::shared_ptr<MemTrackerLimiter> _query_cache_mem_tracker;
466
467
    // TODO, looking forward to more accurate tracking.
468
    std::shared_ptr<MemTrackerLimiter> _rowid_storage_reader_tracker;
469
    std::shared_ptr<MemTrackerLimiter> _subcolumns_tree_tracker;
470
    std::shared_ptr<MemTrackerLimiter> _s3_file_buffer_tracker;
471
472
    // Tracking memory consumption of parquet meta
473
    std::shared_ptr<MemTrackerLimiter> _parquet_meta_tracker;
474
475
    std::unique_ptr<ThreadPool> _send_batch_thread_pool;
476
    // Threadpool used to prefetch remote file for buffered reader
477
    std::unique_ptr<ThreadPool> _buffered_reader_prefetch_thread_pool;
478
    // Threadpool used to send TableStats to FE
479
    std::unique_ptr<ThreadPool> _send_table_stats_thread_pool;
480
    // Threadpool used to upload local file to s3
481
    std::unique_ptr<ThreadPool> _s3_file_upload_thread_pool;
482
    // Pool used by join node to build hash table
483
    // Pool to use a new thread to release object
484
    std::unique_ptr<ThreadPool> _lazy_release_obj_pool;
485
    std::unique_ptr<ThreadPool> _non_block_close_thread_pool;
486
    std::unique_ptr<ThreadPool> _s3_file_system_thread_pool;
487
    // for java-udf to close
488
    std::unique_ptr<ThreadPool> _udf_close_workers_thread_pool;
489
490
    FragmentMgr* _fragment_mgr = nullptr;
491
    WorkloadGroupMgr* _workload_group_manager = nullptr;
492
493
    ResultCache* _result_cache = nullptr;
494
    ClusterInfo* _cluster_info = nullptr;
495
    LoadPathMgr* _load_path_mgr = nullptr;
496
497
    BfdParser* _bfd_parser = nullptr;
498
    BrokerMgr* _broker_mgr = nullptr;
499
    LoadChannelMgr* _load_channel_mgr = nullptr;
500
    std::unique_ptr<LoadStreamMgr> _load_stream_mgr;
501
    std::unique_ptr<NewLoadStreamMgr> _new_load_stream_mgr;
502
    BrpcClientCache<PBackendService_Stub>* _internal_client_cache = nullptr;
503
    BrpcClientCache<PBackendService_Stub>* _streaming_client_cache = nullptr;
504
    BrpcClientCache<PFunctionService_Stub>* _function_client_cache = nullptr;
505
506
    std::unique_ptr<StreamLoadExecutor> _stream_load_executor;
507
    RoutineLoadTaskExecutor* _routine_load_task_executor = nullptr;
508
    StreamLoadRecorderManager* _stream_load_recorder_manager = nullptr;
509
    SmallFileMgr* _small_file_mgr = nullptr;
510
    HeartbeatFlags* _heartbeat_flags = nullptr;
511
512
    // To save meta info of external file, such as parquet footer.
513
    FileMetaCache* _file_meta_cache = nullptr;
514
    std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter;
515
    std::unique_ptr<LoadStreamMapPool> _load_stream_map_pool;
516
    std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool;
517
    std::unique_ptr<WalManager> _wal_manager;
518
    DNSCache* _dns_cache = nullptr;
519
    std::unique_ptr<WriteCooldownMetaExecutors> _write_cooldown_meta_executors;
520
521
    std::mutex _frontends_lock;
522
    // ip:brpc_port -> frontend_indo
523
    std::map<TNetworkAddress, FrontendInfo> _frontends;
524
    GroupCommitMgr* _group_commit_mgr = nullptr;
525
    CdcClientMgr* _cdc_client_mgr = nullptr;
526
527
    // Maybe we should use unique_ptr, but it need complete type, which means we need
528
    // to include many headers, and for some cpp file that do not need class like TabletSchemaCache,
529
    // these redundancy header could introduce potential bug, at least, more header means slow compile.
530
    // So we choose to use raw pointer, please remember to delete these pointer in deconstructor.
531
    TabletSchemaCache* _tablet_schema_cache = nullptr;
532
    TabletColumnObjectPool* _tablet_column_object_pool = nullptr;
533
    std::unique_ptr<BaseStorageEngine> _storage_engine;
534
    SchemaCache* _schema_cache = nullptr;
535
    StoragePageCache* _storage_page_cache = nullptr;
536
    SegmentLoader* _segment_loader = nullptr;
537
    LookupConnectionCache* _lookup_connection_cache = nullptr;
538
    RowCache* _row_cache = nullptr;
539
    CacheManager* _cache_manager = nullptr;
540
    IdManager* _id_manager = nullptr;
541
    ProcessProfile* _process_profile = nullptr;
542
    HeapProfiler* _heap_profiler = nullptr;
543
    segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr;
544
    segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr;
545
    segment_v2::ConditionCache* _condition_cache = nullptr;
546
    segment_v2::EncodingInfoResolver* _encoding_info_resolver = nullptr;
547
    QueryCache* _query_cache = nullptr;
548
    std::unique_ptr<io::FDCache> _file_cache_open_fd_cache;
549
    DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr};
550
551
    pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr;
552
    vectorized::DictionaryFactory* _dict_factory = nullptr;
553
554
    WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr;
555
    IndexPolicyMgr* _index_policy_mgr = nullptr;
556
557
    RuntimeQueryStatisticsMgr* _runtime_query_statistics_mgr = nullptr;
558
559
    std::unique_ptr<pipeline::PipelineTracerContext> _pipeline_tracer_ctx;
560
    std::unique_ptr<segment_v2::TmpFileDirs> _tmp_file_dirs;
561
    doris::vectorized::SpillStreamManager* _spill_stream_mgr = nullptr;
562
563
    orc::MemoryPool* _orc_memory_pool = nullptr;
564
    arrow::MemoryPool* _arrow_memory_pool = nullptr;
565
566
    kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr;
567
    io::HdfsMgr* _hdfs_mgr = nullptr;
568
    io::PackedFileManager* _packed_file_manager = nullptr;
569
};
570
571
template <>
572
0
inline ClientCache<BackendServiceClient>* ExecEnv::get_client_cache<BackendServiceClient>() {
573
0
    return _backend_client_cache;
574
0
}
575
template <>
576
0
inline ClientCache<FrontendServiceClient>* ExecEnv::get_client_cache<FrontendServiceClient>() {
577
0
    return _frontend_client_cache;
578
0
}
579
template <>
580
inline ClientCache<TPaloBrokerServiceClient>*
581
0
ExecEnv::get_client_cache<TPaloBrokerServiceClient>() {
582
0
    return _broker_client_cache;
583
0
}
584
585
0
inline segment_v2::InvertedIndexQueryCache* GetInvertedIndexQueryCache() {
586
0
    return ExecEnv::GetInstance()->get_inverted_index_query_cache();
587
0
}
588
589
} // namespace doris