Coverage Report

Created: 2026-08-11 00:57

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/runtime/query_context.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "runtime/query_context.h"
19
20
#include <fmt/core.h>
21
#include <gen_cpp/FrontendService_types.h>
22
#include <gen_cpp/RuntimeProfile_types.h>
23
#include <gen_cpp/Types_types.h>
24
#include <glog/logging.h>
25
26
#include <algorithm>
27
#include <exception>
28
#include <memory>
29
#include <mutex>
30
#include <utility>
31
#include <vector>
32
33
#include "cloud/config.h"
34
#include "common/logging.h"
35
#include "common/status.h"
36
#include "exec/operator/rec_cte_scan_operator.h"
37
#include "exec/pipeline/dependency.h"
38
#include "exec/pipeline/pipeline_fragment_context.h"
39
#include "exec/runtime_filter/runtime_filter_definitions.h"
40
#include "exec/spill/spill_file_manager.h"
41
#include "io/cache/remote_scan_cache_write_limiter.h"
42
#include "runtime/exec_env.h"
43
#include "runtime/fragment_mgr.h"
44
#include "runtime/memory/heap_profiler.h"
45
#include "runtime/runtime_query_statistics_mgr.h"
46
#include "runtime/runtime_state.h"
47
#include "runtime/thread_context.h"
48
#include "runtime/workload_group/workload_group_manager.h"
49
#include "runtime/workload_management/query_task_controller.h"
50
#include "storage/olap_common.h"
51
#include "util/mem_info.h"
52
#include "util/uid_util.h"
53
54
namespace doris {
55
56
class DelayReleaseToken : public Runnable {
57
    ENABLE_FACTORY_CREATOR(DelayReleaseToken);
58
59
public:
60
0
    DelayReleaseToken(std::unique_ptr<ThreadPoolToken>&& token) { token_ = std::move(token); }
61
    ~DelayReleaseToken() override = default;
62
0
    void run() override {}
63
    std::unique_ptr<ThreadPoolToken> token_;
64
};
65
66
0
const std::string toString(QuerySource queryType) {
67
0
    switch (queryType) {
68
0
    case QuerySource::INTERNAL_FRONTEND:
69
0
        return "INTERNAL_FRONTEND";
70
0
    case QuerySource::STREAM_LOAD:
71
0
        return "STREAM_LOAD";
72
0
    case QuerySource::GROUP_COMMIT_LOAD:
73
0
        return "EXTERNAL_QUERY";
74
0
    case QuerySource::ROUTINE_LOAD:
75
0
        return "ROUTINE_LOAD";
76
0
    case QuerySource::EXTERNAL_CONNECTOR:
77
0
        return "EXTERNAL_CONNECTOR";
78
0
    case QuerySource::EXTERNAL_FRONTEND:
79
0
        return "EXTERNAL_FRONTEND";
80
0
    default:
81
0
        return "UNKNOWN";
82
0
    }
83
0
}
84
85
std::shared_ptr<QueryContext> QueryContext::create(TUniqueId query_id, ExecEnv* exec_env,
86
                                                   const TQueryOptions& query_options,
87
                                                   TNetworkAddress coord_addr, bool is_nereids,
88
                                                   TNetworkAddress current_connect_fe,
89
149
                                                   QuerySource query_type) {
90
149
    auto ctx = QueryContext::create_shared(query_id, exec_env, query_options, coord_addr,
91
149
                                           is_nereids, current_connect_fe, query_type);
92
149
    ctx->init_query_task_controller();
93
149
    return ctx;
94
149
}
95
96
QueryContext::QueryContext(TUniqueId query_id, ExecEnv* exec_env,
97
                           const TQueryOptions& query_options, TNetworkAddress coord_addr,
98
                           bool is_nereids, TNetworkAddress current_connect_fe,
99
                           QuerySource query_source)
100
122k
        : _timeout_second(-1),
101
122k
          _query_id(std::move(query_id)),
102
122k
          _exec_env(exec_env),
103
122k
          _is_nereids(is_nereids),
104
122k
          _query_options(query_options),
105
122k
          _query_source(query_source) {
106
122k
    _init_resource_context();
107
122k
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_mem_tracker());
108
122k
    _query_watcher.start();
109
122k
    _execution_dependency = Dependency::create_unique(-1, -1, "ExecutionDependency", false);
110
122k
    _memory_sufficient_dependency =
111
122k
            Dependency::create_unique(-1, -1, "MemorySufficientDependency", true);
112
113
122k
    _runtime_filter_mgr = std::make_unique<RuntimeFilterMgr>(true);
114
115
122k
    _timeout_second = query_options.execution_timeout;
116
117
122k
    bool initialize_context_holder =
118
122k
            config::enable_file_cache && config::enable_file_cache_query_limit &&
119
122k
            query_options.__isset.enable_file_cache && query_options.enable_file_cache &&
120
122k
            query_options.__isset.file_cache_query_limit_percent &&
121
122k
            query_options.file_cache_query_limit_percent < 100;
122
123
    // Init query context holders for file cache, if enable query limit feature
124
122k
    if (initialize_context_holder) {
125
0
        _query_context_holders = io::FileCacheFactory::instance()->get_query_context_holders(
126
0
                _query_id, query_options.file_cache_query_limit_percent);
127
0
    }
128
129
122k
    const bool initialize_remote_scan_cache_write_limiter =
130
122k
            config::is_cloud_mode() && config::enable_file_cache &&
131
122k
            query_options.__isset.file_cache_query_limit_bytes &&
132
122k
            query_options.file_cache_query_limit_bytes >= 0 &&
133
122k
            query_options.query_type == TQueryType::SELECT;
134
122k
    if (initialize_remote_scan_cache_write_limiter) {
135
3
        _remote_scan_cache_write_limiter = std::make_unique<io::RemoteScanCacheWriteLimiter>(
136
3
                _query_id, query_options.file_cache_query_limit_bytes);
137
3
    }
138
139
122k
    bool is_query_type_valid = query_options.query_type == TQueryType::SELECT ||
140
122k
                               query_options.query_type == TQueryType::LOAD ||
141
122k
                               query_options.query_type == TQueryType::EXTERNAL;
142
122k
    DCHECK_EQ(is_query_type_valid, true);
143
144
122k
    this->coord_addr = coord_addr;
145
    // current_connect_fe is used for report query statistics
146
122k
    this->current_connect_fe = current_connect_fe;
147
    // external query has no current_connect_fe
148
122k
    if (query_options.query_type != TQueryType::EXTERNAL) {
149
301
        bool is_report_fe_addr_valid =
150
301
                !this->current_connect_fe.hostname.empty() && this->current_connect_fe.port != 0;
151
301
        DCHECK_EQ(is_report_fe_addr_valid, true);
152
301
    }
153
122k
    clock_gettime(CLOCK_MONOTONIC, &this->_query_arrival_timestamp);
154
122k
    DorisMetrics::instance()->query_ctx_cnt->increment(1);
155
122k
}
156
157
122k
void QueryContext::_init_query_mem_tracker() {
158
    // If user not set query limit, will use default 1TB memory limit. It is large enough to cover most cases.
159
122k
    constexpr int64_t DEFAULT_QUERY_MEM_LIMIT = 1LL << 60;
160
122k
    bool has_query_mem_limit = _query_options.__isset.mem_limit && (_query_options.mem_limit > 0);
161
122k
    int64_t user_set_mem_limit =
162
122k
            has_query_mem_limit ? _query_options.mem_limit : DEFAULT_QUERY_MEM_LIMIT;
163
122k
    int64_t adjusted_mem_limit = user_set_mem_limit;
164
122k
    if (adjusted_mem_limit > MemInfo::mem_limit()) {
165
39
        VLOG_NOTICE << "Query memory limit "
166
0
                    << PrettyPrinter::print(user_set_mem_limit, TUnit::BYTES)
167
0
                    << " exceeds process memory limit of "
168
0
                    << PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES)
169
0
                    << ". Using process memory limit instead.";
170
39
        adjusted_mem_limit = MemInfo::mem_limit();
171
39
    }
172
    // If the query is a pure load task(streamload, routine load, group commit), then it should not use
173
    // memlimit per query to limit their memory usage.
174
122k
    if (is_pure_load_task()) {
175
122k
        adjusted_mem_limit = MemInfo::mem_limit();
176
122k
    }
177
122k
    std::shared_ptr<MemTrackerLimiter> query_mem_tracker;
178
122k
    if (_query_options.query_type == TQueryType::SELECT) {
179
298
        query_mem_tracker = MemTrackerLimiter::create_shared(
180
298
                MemTrackerLimiter::Type::QUERY, fmt::format("Query#Id={}", print_id(_query_id)),
181
298
                adjusted_mem_limit);
182
122k
    } else if (_query_options.query_type == TQueryType::LOAD) {
183
3
        query_mem_tracker = MemTrackerLimiter::create_shared(
184
3
                MemTrackerLimiter::Type::LOAD, fmt::format("Load#Id={}", print_id(_query_id)),
185
3
                adjusted_mem_limit);
186
122k
    } else if (_query_options.query_type == TQueryType::EXTERNAL) { // spark/flink/etc..
187
122k
        query_mem_tracker = MemTrackerLimiter::create_shared(
188
122k
                MemTrackerLimiter::Type::QUERY, fmt::format("External#Id={}", print_id(_query_id)),
189
122k
                adjusted_mem_limit);
190
122k
    } else {
191
0
        LOG(FATAL) << "__builtin_unreachable";
192
0
        __builtin_unreachable();
193
0
    }
194
122k
    if (_query_options.__isset.is_report_success && _query_options.is_report_success) {
195
0
        query_mem_tracker->enable_print_log_usage();
196
0
    }
197
198
    // If enable reserve memory, not enable check limit, because reserve memory will check it.
199
    // If reserve enabled, even if the reserved memory size is smaller than the actual requested memory,
200
    // and the query memory consumption is larger than the limit, we do not expect the query to fail
201
    // after `check_limit` returns an error, but to run as long as possible,
202
    // and will enter the paused state and try to spill when the query reserves next time.
203
    // If the workload group or process runs out of memory, it will be forced to cancel.
204
122k
    query_mem_tracker->set_enable_check_limit(!(_query_options.__isset.enable_reserve_memory &&
205
122k
                                                _query_options.enable_reserve_memory));
206
122k
    _resource_ctx->memory_context()->set_mem_tracker(query_mem_tracker);
207
122k
    _resource_ctx->memory_context()->set_user_set_mem_limit(user_set_mem_limit);
208
122k
}
209
210
122k
void QueryContext::_init_resource_context() {
211
122k
    _resource_ctx = ResourceContext::create_shared();
212
122k
    _init_query_mem_tracker();
213
122k
}
214
215
122k
void QueryContext::init_query_task_controller() {
216
122k
    _resource_ctx->set_task_controller(QueryTaskController::create(shared_from_this()));
217
122k
    _resource_ctx->task_controller()->set_task_id(_query_id);
218
122k
    _resource_ctx->task_controller()->set_fe_addr(current_connect_fe);
219
122k
    _resource_ctx->task_controller()->set_query_type(_query_options.query_type);
220
#ifndef BE_TEST
221
    _exec_env->runtime_query_statistics_mgr()->register_resource_context(print_id(_query_id),
222
                                                                         _resource_ctx);
223
#endif
224
122k
}
225
226
251
void QueryContext::record_spill_data_dir(SpillDataDir* data_dir) {
227
251
    std::lock_guard lock(_spill_data_dirs_mutex);
228
251
    _spill_data_dirs.emplace(data_dir);
229
251
}
230
231
122k
QueryContext::~QueryContext() {
232
122k
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_mem_tracker());
233
    // query mem tracker consumption is equal to 0, it means that after QueryContext is created,
234
    // it is found that query already exists in _query_ctx_map, and query mem tracker is not used.
235
    // query mem tracker consumption is not equal to 0 after use, because there is memory consumed
236
    // on query mem tracker, released on other trackers.
237
122k
    std::string mem_tracker_msg;
238
122k
    if (query_mem_tracker()->peak_consumption() != 0) {
239
35
        mem_tracker_msg = fmt::format(
240
35
                "deregister query/load memory tracker, queryId={}, Limit={}, CurrUsed={}, "
241
35
                "PeakUsed={}",
242
35
                print_id(_query_id), PrettyPrinter::print_bytes(query_mem_tracker()->limit()),
243
35
                PrettyPrinter::print_bytes(query_mem_tracker()->consumption()),
244
35
                PrettyPrinter::print_bytes(query_mem_tracker()->peak_consumption()));
245
35
    }
246
122k
    [[maybe_unused]] uint64_t group_id = 0;
247
122k
    if (workload_group()) {
248
46
        group_id = workload_group()->id(); // before remove
249
46
    }
250
251
122k
    _resource_ctx->task_controller()->finish();
252
253
122k
    if (enable_profile()) {
254
0
        _report_query_profile();
255
0
    }
256
257
#ifndef BE_TEST
258
    if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) [[unlikely]] {
259
        try {
260
            ExecEnv::GetInstance()->pipeline_tracer_context()->end_query(_query_id, group_id);
261
        } catch (std::exception& e) {
262
            LOG(WARNING) << "Dump trace log failed bacause " << e.what();
263
        }
264
    }
265
#endif
266
122k
    _runtime_filter_mgr.reset();
267
122k
    _execution_dependency.reset();
268
122k
    _runtime_predicates.clear();
269
122k
    file_scan_range_params_map.clear();
270
122k
    obj_pool.clear();
271
122k
    _merge_controller_handler.reset();
272
273
122k
    if (auto* spill_file_mgr = _exec_env->spill_file_mgr()) {
274
248
        for (auto* data_dir : _spill_data_dirs) {
275
32
            spill_file_mgr->delete_query_spill_directory(print_id(_query_id), data_dir);
276
32
        }
277
248
    }
278
279
122k
    DorisMetrics::instance()->query_ctx_cnt->increment(-1);
280
    // fragment_mgr is nullptr in unittest
281
122k
    if (ExecEnv::GetInstance()->fragment_mgr()) {
282
1
        ExecEnv::GetInstance()->fragment_mgr()->remove_query_context(this->_query_id);
283
1
    }
284
    // the only one msg shows query's end. any other msg should append to it if need.
285
122k
    timespec now;
286
122k
    clock_gettime(CLOCK_MONOTONIC, &now);
287
122k
    int64_t elapsed_ms = (now.tv_sec - _query_arrival_timestamp.tv_sec) * 1000LL +
288
122k
                         (now.tv_nsec - _query_arrival_timestamp.tv_nsec) / 1000000LL;
289
122k
    LOG_INFO("Query {} deconstructed, elapsed_ms: {}, mem_tracker: {}", print_id(this->_query_id),
290
122k
             elapsed_ms, mem_tracker_msg);
291
122k
}
292
293
25
void QueryContext::set_ready_to_execute(Status reason) {
294
25
    set_execution_dependency_ready();
295
25
    _exec_status.update(reason);
296
25
}
297
298
0
void QueryContext::set_ready_to_execute_only() {
299
0
    set_execution_dependency_ready();
300
0
}
301
302
25
void QueryContext::set_execution_dependency_ready() {
303
25
    _execution_dependency->set_ready();
304
25
}
305
306
46
void QueryContext::set_memory_sufficient(bool sufficient) {
307
46
    if (sufficient) {
308
17
        {
309
17
            _memory_sufficient_dependency->set_ready();
310
17
            _resource_ctx->task_controller()->reset_paused_reason();
311
17
        }
312
29
    } else {
313
29
        _memory_sufficient_dependency->block();
314
29
        _resource_ctx->task_controller()->add_paused_count();
315
29
    }
316
46
}
317
318
25
void QueryContext::cancel(Status new_status, int fragment_id) {
319
25
    if (!_exec_status.update(new_status)) {
320
0
        return;
321
0
    }
322
    // Tasks should be always runnable.
323
25
    _execution_dependency->set_always_ready();
324
25
    _memory_sufficient_dependency->set_always_ready();
325
25
    if ((new_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>() ||
326
25
         new_status.is<ErrorCode::MEM_ALLOC_FAILED>()) &&
327
25
        _query_options.__isset.dump_heap_profile_when_mem_limit_exceeded &&
328
25
        _query_options.dump_heap_profile_when_mem_limit_exceeded) {
329
        // if query is cancelled because of query mem limit exceeded, dump heap profile
330
        // at the time of cancellation can get the most accurate memory usage for problem analysis
331
0
        auto wg = workload_group();
332
0
        auto log_str = fmt::format(
333
0
                "Query {} canceled because of memory limit exceeded, dumping memory "
334
0
                "detail profiles. wg: {}. {}",
335
0
                print_id(_query_id), wg ? wg->debug_string() : "null",
336
0
                doris::ProcessProfile::instance()->memory_profile()->process_memory_detail_str());
337
0
        LOG_LONG_STRING(INFO, log_str);
338
0
        std::string dot = HeapProfiler::instance()->dump_heap_profile_to_dot();
339
0
        if (!dot.empty()) {
340
0
            dot += "\n-------------------------------------------------------\n";
341
0
            dot += "Copy the text after `digraph` in the above output to "
342
0
                   "http://www.webgraphviz.com to generate a dot graph.\n"
343
0
                   "after start heap profiler, if there is no operation, will print `No nodes "
344
0
                   "to "
345
0
                   "print`."
346
0
                   "If there are many errors: `addr2line: Dwarf Error`,"
347
0
                   "or other FAQ, reference doc: "
348
0
                   "https://doris.apache.org/community/developer-guide/debug-tool/#4-qa\n";
349
0
            auto log_str =
350
0
                    fmt::format("Query {}, dump heap profile to dot: {}", print_id(_query_id), dot);
351
0
            LOG_LONG_STRING(INFO, log_str);
352
0
        }
353
0
    }
354
355
25
    set_ready_to_execute(new_status);
356
25
    cancel_all_pipeline_context(new_status, fragment_id);
357
25
}
358
359
0
void QueryContext::set_load_error_url(std::string error_url) {
360
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
361
0
    _load_error_url = error_url;
362
0
}
363
364
0
std::string QueryContext::get_load_error_url() {
365
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
366
0
    return _load_error_url;
367
0
}
368
369
0
void QueryContext::set_first_error_msg(std::string error_msg) {
370
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
371
0
    _first_error_msg = error_msg;
372
0
}
373
374
0
std::string QueryContext::get_first_error_msg() {
375
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
376
0
    return _first_error_msg;
377
0
}
378
379
25
void QueryContext::cancel_all_pipeline_context(const Status& reason, int fragment_id) {
380
25
    std::vector<std::weak_ptr<PipelineFragmentContext>> ctx_to_cancel;
381
25
    {
382
25
        std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
383
25
        for (auto& [f_id, f_context] : _fragment_id_to_pipeline_ctx) {
384
0
            if (fragment_id == f_id) {
385
0
                continue;
386
0
            }
387
0
            ctx_to_cancel.push_back(f_context);
388
0
        }
389
25
    }
390
25
    for (auto& f_context : ctx_to_cancel) {
391
0
        if (auto pipeline_ctx = f_context.lock()) {
392
0
            pipeline_ctx->cancel(reason);
393
0
        }
394
0
    }
395
25
}
396
397
0
std::string QueryContext::print_all_pipeline_context() {
398
0
    std::vector<std::weak_ptr<PipelineFragmentContext>> ctx_to_print;
399
0
    fmt::memory_buffer debug_string_buffer;
400
0
    size_t i = 0;
401
0
    {
402
0
        fmt::format_to(debug_string_buffer, "{} pipeline fragment contexts in query {}. \n",
403
0
                       _fragment_id_to_pipeline_ctx.size(), print_id(_query_id));
404
405
0
        {
406
0
            std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
407
0
            for (auto& [f_id, f_context] : _fragment_id_to_pipeline_ctx) {
408
0
                ctx_to_print.push_back(f_context);
409
0
            }
410
0
        }
411
0
        for (auto& f_context : ctx_to_print) {
412
0
            if (auto pipeline_ctx = f_context.lock()) {
413
0
                auto elapsed = pipeline_ctx->elapsed_time() / 1000000000.0;
414
0
                fmt::format_to(debug_string_buffer,
415
0
                               "No.{} (elapse_second={}s, fragment_id={}) : {}\n", i, elapsed,
416
0
                               pipeline_ctx->get_fragment_id(), pipeline_ctx->debug_string());
417
0
                i++;
418
0
            }
419
0
        }
420
0
    }
421
0
    return fmt::to_string(debug_string_buffer);
422
0
}
423
424
void QueryContext::set_pipeline_context(const int fragment_id,
425
0
                                        std::shared_ptr<PipelineFragmentContext> pip_ctx) {
426
0
    std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
427
    // Use insert_or_assign instead of insert to support overwriting old entries
428
    // when recursive CTE recreates PipelineFragmentContext between rounds.
429
0
    _fragment_id_to_pipeline_ctx.insert_or_assign(fragment_id, pip_ctx);
430
0
}
431
432
30
doris::TaskScheduler* QueryContext::get_pipe_exec_scheduler() {
433
30
    if (!_task_scheduler) {
434
0
        throw Exception(Status::InternalError("task_scheduler is null"));
435
0
    }
436
30
    return _task_scheduler;
437
30
}
438
439
45
Status QueryContext::set_workload_group(WorkloadGroupPtr& wg) {
440
45
    _resource_ctx->set_workload_group(wg);
441
    // Should add query first, the workload group will not be deleted,
442
    // then visit workload group's resource
443
    // see task_group_manager::delete_workload_group_by_ids
444
45
    RETURN_IF_ERROR(workload_group()->add_resource_ctx(_query_id, _resource_ctx));
445
446
45
    workload_group()->get_query_scheduler(&_task_scheduler, &_scan_task_scheduler,
447
45
                                          &_remote_scan_task_scheduler);
448
45
    return Status::OK();
449
45
}
450
451
void QueryContext::add_fragment_profile(
452
        int fragment_id, const std::vector<std::shared_ptr<TRuntimeProfileTree>>& pipeline_profiles,
453
0
        std::shared_ptr<TRuntimeProfileTree> load_channel_profile) {
454
0
    if (pipeline_profiles.empty()) {
455
0
        std::string msg = fmt::format("Add pipeline profile failed, query {}, fragment {}",
456
0
                                      print_id(this->_query_id), fragment_id);
457
0
        LOG_ERROR(msg);
458
0
        DCHECK(false) << msg;
459
0
        return;
460
0
    }
461
462
0
#ifndef NDEBUG
463
0
    for (const auto& p : pipeline_profiles) {
464
0
        DCHECK(p != nullptr) << fmt::format("Add pipeline profile failed, query {}, fragment {}",
465
0
                                            print_id(this->_query_id), fragment_id);
466
0
    }
467
0
#endif
468
469
0
    std::lock_guard<std::mutex> l(_profile_mutex);
470
0
    VLOG_ROW << fmt::format(
471
0
            "Query add fragment profile, query {}, fragment {}, pipeline profile count {} ",
472
0
            print_id(this->_query_id), fragment_id, pipeline_profiles.size());
473
474
0
    _profile_map.insert(std::make_pair(fragment_id, pipeline_profiles));
475
476
0
    if (load_channel_profile != nullptr) {
477
0
        _load_channel_profile_map.insert(std::make_pair(fragment_id, load_channel_profile));
478
0
    }
479
0
}
480
481
0
void QueryContext::_report_query_profile() {
482
0
    std::lock_guard<std::mutex> lg(_profile_mutex);
483
484
0
    for (auto& [fragment_id, fragment_profile] : _profile_map) {
485
0
        std::shared_ptr<TRuntimeProfileTree> load_channel_profile = nullptr;
486
487
0
        if (_load_channel_profile_map.contains(fragment_id)) {
488
0
            load_channel_profile = _load_channel_profile_map[fragment_id];
489
0
        }
490
491
0
        ExecEnv::GetInstance()->runtime_query_statistics_mgr()->register_fragment_profile(
492
0
                _query_id, this->coord_addr, fragment_id, fragment_profile, load_channel_profile);
493
0
    }
494
495
0
    ExecEnv::GetInstance()->runtime_query_statistics_mgr()->trigger_profile_reporting();
496
0
}
497
498
std::unordered_map<int, std::vector<std::shared_ptr<TRuntimeProfileTree>>>
499
0
QueryContext::_collect_realtime_query_profile() {
500
0
    std::unordered_map<int, std::vector<std::shared_ptr<TRuntimeProfileTree>>> res;
501
0
    std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
502
0
    for (const auto& [fragment_id, fragment_ctx_wptr] : _fragment_id_to_pipeline_ctx) {
503
0
        if (auto fragment_ctx = fragment_ctx_wptr.lock()) {
504
0
            if (fragment_ctx == nullptr) {
505
0
                std::string msg =
506
0
                        fmt::format("PipelineFragmentContext is nullptr, query {} fragment_id: {}",
507
0
                                    print_id(_query_id), fragment_id);
508
0
                LOG_ERROR(msg);
509
0
                DCHECK(false) << msg;
510
0
                continue;
511
0
            }
512
513
0
            auto profile = fragment_ctx->collect_realtime_profile();
514
515
0
            if (profile.empty()) {
516
0
                std::string err_msg = fmt::format(
517
0
                        "Get nothing when collecting profile, query {}, fragment_id: {}",
518
0
                        print_id(_query_id), fragment_id);
519
0
                LOG_ERROR(err_msg);
520
0
                DCHECK(false) << err_msg;
521
0
                continue;
522
0
            }
523
524
0
            res.insert(std::make_pair(fragment_id, profile));
525
0
        }
526
0
    }
527
528
0
    return res;
529
0
}
530
531
0
TReportExecStatusParams QueryContext::get_realtime_exec_status() {
532
0
    TReportExecStatusParams exec_status;
533
534
0
    auto realtime_query_profile = _collect_realtime_query_profile();
535
0
    std::vector<std::shared_ptr<TRuntimeProfileTree>> load_channel_profiles;
536
537
0
    for (auto load_channel_profile : _load_channel_profile_map) {
538
0
        if (load_channel_profile.second != nullptr) {
539
0
            load_channel_profiles.push_back(load_channel_profile.second);
540
0
        }
541
0
    }
542
543
0
    exec_status = RuntimeQueryStatisticsMgr::create_report_exec_status_params(
544
0
            this->_query_id, std::move(realtime_query_profile), std::move(load_channel_profiles),
545
0
            /*is_done=*/false);
546
547
0
    return exec_status;
548
0
}
549
550
Status QueryContext::send_block_to_cte_scan(
551
        const TUniqueId& instance_id, int node_id,
552
0
        const google::protobuf::RepeatedPtrField<doris::PBlock>& pblocks, bool eos) {
553
0
    std::unique_lock<std::mutex> l(_cte_scan_lock);
554
0
    auto it = _cte_scan.find(std::make_pair(instance_id, node_id));
555
0
    if (it == _cte_scan.end()) {
556
0
        return Status::InternalError("RecCTEScan not found for instance {}, node {}",
557
0
                                     print_id(instance_id), node_id);
558
0
    }
559
0
    for (const auto& pblock : pblocks) {
560
0
        RETURN_IF_ERROR(it->second->add_block(pblock));
561
0
    }
562
0
    if (eos) {
563
0
        it->second->set_ready();
564
0
    }
565
0
    return Status::OK();
566
0
}
567
568
void QueryContext::registe_cte_scan(const TUniqueId& instance_id, int node_id,
569
0
                                    RecCTEScanLocalState* scan) {
570
0
    std::unique_lock<std::mutex> l(_cte_scan_lock);
571
0
    auto key = std::make_pair(instance_id, node_id);
572
0
    DCHECK(!_cte_scan.contains(key)) << "Duplicate registe cte scan for instance "
573
0
                                     << print_id(instance_id) << ", node " << node_id;
574
0
    _cte_scan.emplace(key, scan);
575
0
}
576
577
0
void QueryContext::deregiste_cte_scan(const TUniqueId& instance_id, int node_id) {
578
0
    std::lock_guard<std::mutex> l(_cte_scan_lock);
579
0
    auto key = std::make_pair(instance_id, node_id);
580
0
    DCHECK(_cte_scan.contains(key)) << "Duplicate deregiste cte scan for instance "
581
0
                                    << print_id(instance_id) << ", node " << node_id;
582
0
    _cte_scan.erase(key);
583
0
}
584
585
0
Status QueryContext::reset_global_rf(const google::protobuf::RepeatedField<int32_t>& filter_ids) {
586
0
    if (_merge_controller_handler) {
587
0
        return _merge_controller_handler->reset_global_rf(this, filter_ids);
588
0
    }
589
0
    return Status::OK();
590
0
}
591
592
6
void QueryContext::add_total_task_num(int delta) {
593
6
    if (auto* qtc = dynamic_cast<QueryTaskController*>(_resource_ctx->task_controller())) {
594
6
        qtc->add_total_task_num(delta);
595
6
    }
596
6
}
597
598
421
void QueryContext::inc_finished_task_num() {
599
421
    if (auto* qtc = dynamic_cast<QueryTaskController*>(_resource_ctx->task_controller())) {
600
421
        qtc->inc_finished_task_num();
601
421
    }
602
421
}
603
604
} // namespace doris