Coverage Report

Created: 2026-11-05 16:38

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/runtime/query_context.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "runtime/query_context.h"
19
20
#include <fmt/core.h>
21
#include <gen_cpp/FrontendService_types.h>
22
#include <gen_cpp/RuntimeProfile_types.h>
23
#include <gen_cpp/Types_types.h>
24
#include <glog/logging.h>
25
26
#include <algorithm>
27
#include <exception>
28
#include <memory>
29
#include <mutex>
30
#include <utility>
31
#include <vector>
32
33
#include "cloud/config.h"
34
#include "common/logging.h"
35
#include "common/status.h"
36
#include "exec/operator/rec_cte_scan_operator.h"
37
#include "exec/pipeline/dependency.h"
38
#include "exec/pipeline/pipeline_fragment_context.h"
39
#include "exec/runtime_filter/runtime_filter_definitions.h"
40
#include "exec/spill/spill_file_manager.h"
41
#include "exprs/function/ai/ai_request_executor.h"
42
#include "io/cache/remote_scan_cache_write_limiter.h"
43
#include "runtime/exec_env.h"
44
#include "runtime/fragment_mgr.h"
45
#include "runtime/memory/heap_profiler.h"
46
#include "runtime/runtime_query_statistics_mgr.h"
47
#include "runtime/runtime_state.h"
48
#include "runtime/thread_context.h"
49
#include "runtime/workload_group/workload_group_manager.h"
50
#include "runtime/workload_management/query_task_controller.h"
51
#include "storage/olap_common.h"
52
#include "util/mem_info.h"
53
#include "util/uid_util.h"
54
55
namespace doris {
56
57
class DelayReleaseToken : public Runnable {
58
    ENABLE_FACTORY_CREATOR(DelayReleaseToken);
59
60
public:
61
0
    DelayReleaseToken(std::unique_ptr<ThreadPoolToken>&& token) { token_ = std::move(token); }
62
    ~DelayReleaseToken() override = default;
63
0
    void run() override {}
64
    std::unique_ptr<ThreadPoolToken> token_;
65
};
66
67
0
const std::string toString(QuerySource queryType) {
68
0
    switch (queryType) {
69
0
    case QuerySource::INTERNAL_FRONTEND:
70
0
        return "INTERNAL_FRONTEND";
71
0
    case QuerySource::STREAM_LOAD:
72
0
        return "STREAM_LOAD";
73
0
    case QuerySource::GROUP_COMMIT_LOAD:
74
0
        return "EXTERNAL_QUERY";
75
0
    case QuerySource::ROUTINE_LOAD:
76
0
        return "ROUTINE_LOAD";
77
0
    case QuerySource::EXTERNAL_CONNECTOR:
78
0
        return "EXTERNAL_CONNECTOR";
79
0
    case QuerySource::EXTERNAL_FRONTEND:
80
0
        return "EXTERNAL_FRONTEND";
81
0
    default:
82
0
        return "UNKNOWN";
83
0
    }
84
0
}
85
86
std::shared_ptr<QueryContext> QueryContext::create(TUniqueId query_id, ExecEnv* exec_env,
87
                                                   const TQueryOptions& query_options,
88
                                                   TNetworkAddress coord_addr, bool is_nereids,
89
                                                   TNetworkAddress current_connect_fe,
90
189
                                                   QuerySource query_type) {
91
189
    auto ctx = QueryContext::create_shared(query_id, exec_env, query_options, coord_addr,
92
189
                                           is_nereids, current_connect_fe, query_type);
93
189
    ctx->init_query_task_controller();
94
189
    return ctx;
95
189
}
96
97
QueryContext::QueryContext(TUniqueId query_id, ExecEnv* exec_env,
98
                           const TQueryOptions& query_options, TNetworkAddress coord_addr,
99
                           bool is_nereids, TNetworkAddress current_connect_fe,
100
                           QuerySource query_source)
101
127k
        : _timeout_second(-1),
102
127k
          _query_id(std::move(query_id)),
103
127k
          _exec_env(exec_env),
104
127k
          _is_nereids(is_nereids),
105
127k
          _query_options(query_options),
106
127k
          _query_source(query_source) {
107
127k
    _init_resource_context();
108
127k
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_mem_tracker());
109
127k
    _query_watcher.start();
110
127k
    _execution_dependency = Dependency::create_unique(-1, -1, "ExecutionDependency", false);
111
127k
    _memory_sufficient_dependency =
112
127k
            Dependency::create_unique(-1, -1, "MemorySufficientDependency", true);
113
114
127k
    _runtime_filter_mgr = std::make_unique<RuntimeFilterMgr>(true);
115
116
127k
    _timeout_second = query_options.execution_timeout;
117
118
127k
    bool initialize_context_holder =
119
127k
            config::enable_file_cache && config::enable_file_cache_query_limit &&
120
127k
            query_options.__isset.enable_file_cache && query_options.enable_file_cache &&
121
127k
            query_options.__isset.file_cache_query_limit_percent &&
122
127k
            query_options.file_cache_query_limit_percent < 100;
123
124
    // Init query context holders for file cache, if enable query limit feature
125
127k
    if (initialize_context_holder) {
126
0
        _query_context_holders = io::FileCacheFactory::instance()->get_query_context_holders(
127
0
                _query_id, query_options.file_cache_query_limit_percent);
128
0
    }
129
130
127k
    const bool initialize_remote_scan_cache_write_limiter =
131
127k
            config::is_cloud_mode() && config::enable_file_cache &&
132
127k
            query_options.__isset.file_cache_query_limit_bytes &&
133
127k
            query_options.file_cache_query_limit_bytes >= 0 &&
134
127k
            query_options.query_type == TQueryType::SELECT;
135
127k
    if (initialize_remote_scan_cache_write_limiter) {
136
3
        _remote_scan_cache_write_limiter = std::make_unique<io::RemoteScanCacheWriteLimiter>(
137
3
                _query_id, query_options.file_cache_query_limit_bytes);
138
3
    }
139
140
127k
    bool is_query_type_valid = query_options.query_type == TQueryType::SELECT ||
141
127k
                               query_options.query_type == TQueryType::LOAD ||
142
127k
                               query_options.query_type == TQueryType::EXTERNAL;
143
127k
    DCHECK_EQ(is_query_type_valid, true);
144
145
127k
    this->coord_addr = coord_addr;
146
    // current_connect_fe is used for report query statistics
147
127k
    this->current_connect_fe = current_connect_fe;
148
    // external query has no current_connect_fe
149
127k
    if (query_options.query_type != TQueryType::EXTERNAL) {
150
343
        bool is_report_fe_addr_valid =
151
343
                !this->current_connect_fe.hostname.empty() && this->current_connect_fe.port != 0;
152
343
        DCHECK_EQ(is_report_fe_addr_valid, true);
153
343
    }
154
127k
    clock_gettime(CLOCK_MONOTONIC, &this->_query_arrival_timestamp);
155
127k
    DorisMetrics::instance()->query_ctx_cnt->increment(1);
156
127k
}
157
158
127k
void QueryContext::_init_query_mem_tracker() {
159
    // If user not set query limit, will use default 1TB memory limit. It is large enough to cover most cases.
160
127k
    constexpr int64_t DEFAULT_QUERY_MEM_LIMIT = 1LL << 60;
161
127k
    bool has_query_mem_limit = _query_options.__isset.mem_limit && (_query_options.mem_limit > 0);
162
127k
    int64_t user_set_mem_limit =
163
127k
            has_query_mem_limit ? _query_options.mem_limit : DEFAULT_QUERY_MEM_LIMIT;
164
127k
    int64_t adjusted_mem_limit = user_set_mem_limit;
165
127k
    if (adjusted_mem_limit > MemInfo::mem_limit()) {
166
39
        VLOG_NOTICE << "Query memory limit "
167
0
                    << PrettyPrinter::print(user_set_mem_limit, TUnit::BYTES)
168
0
                    << " exceeds process memory limit of "
169
0
                    << PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES)
170
0
                    << ". Using process memory limit instead.";
171
39
        adjusted_mem_limit = MemInfo::mem_limit();
172
39
    }
173
    // If the query is a pure load task(streamload, routine load, group commit), then it should not use
174
    // memlimit per query to limit their memory usage.
175
127k
    if (is_pure_load_task()) {
176
127k
        adjusted_mem_limit = MemInfo::mem_limit();
177
127k
    }
178
127k
    std::shared_ptr<MemTrackerLimiter> query_mem_tracker;
179
127k
    if (_query_options.query_type == TQueryType::SELECT) {
180
340
        query_mem_tracker = MemTrackerLimiter::create_shared(
181
340
                MemTrackerLimiter::Type::QUERY, fmt::format("Query#Id={}", print_id(_query_id)),
182
340
                adjusted_mem_limit);
183
127k
    } else if (_query_options.query_type == TQueryType::LOAD) {
184
3
        query_mem_tracker = MemTrackerLimiter::create_shared(
185
3
                MemTrackerLimiter::Type::LOAD, fmt::format("Load#Id={}", print_id(_query_id)),
186
3
                adjusted_mem_limit);
187
127k
    } else if (_query_options.query_type == TQueryType::EXTERNAL) { // spark/flink/etc..
188
127k
        query_mem_tracker = MemTrackerLimiter::create_shared(
189
127k
                MemTrackerLimiter::Type::QUERY, fmt::format("External#Id={}", print_id(_query_id)),
190
127k
                adjusted_mem_limit);
191
127k
    } else {
192
0
        LOG(FATAL) << "__builtin_unreachable";
193
0
        __builtin_unreachable();
194
0
    }
195
127k
    if (_query_options.__isset.is_report_success && _query_options.is_report_success) {
196
0
        query_mem_tracker->enable_print_log_usage();
197
0
    }
198
199
    // If enable reserve memory, not enable check limit, because reserve memory will check it.
200
    // If reserve enabled, even if the reserved memory size is smaller than the actual requested memory,
201
    // and the query memory consumption is larger than the limit, we do not expect the query to fail
202
    // after `check_limit` returns an error, but to run as long as possible,
203
    // and will enter the paused state and try to spill when the query reserves next time.
204
    // If the workload group or process runs out of memory, it will be forced to cancel.
205
127k
    query_mem_tracker->set_enable_check_limit(!(_query_options.__isset.enable_reserve_memory &&
206
127k
                                                _query_options.enable_reserve_memory));
207
127k
    _resource_ctx->memory_context()->set_mem_tracker(query_mem_tracker);
208
127k
    _resource_ctx->memory_context()->set_user_set_mem_limit(user_set_mem_limit);
209
127k
}
210
211
127k
void QueryContext::_init_resource_context() {
212
127k
    _resource_ctx = ResourceContext::create_shared();
213
127k
    _init_query_mem_tracker();
214
127k
}
215
216
127k
void QueryContext::init_query_task_controller() {
217
127k
    _resource_ctx->set_task_controller(QueryTaskController::create(shared_from_this()));
218
127k
    _resource_ctx->task_controller()->set_task_id(_query_id);
219
127k
    _resource_ctx->task_controller()->set_fe_addr(current_connect_fe);
220
127k
    _resource_ctx->task_controller()->set_query_type(_query_options.query_type);
221
#ifndef BE_TEST
222
    _exec_env->runtime_query_statistics_mgr()->register_resource_context(print_id(_query_id),
223
                                                                         _resource_ctx);
224
#endif
225
127k
}
226
227
255
void QueryContext::record_spill_data_dir(SpillDataDir* data_dir) {
228
255
    std::lock_guard lock(_spill_data_dirs_mutex);
229
255
    _spill_data_dirs.emplace(data_dir);
230
255
}
231
232
127k
QueryContext::~QueryContext() {
233
127k
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_mem_tracker());
234
    // query mem tracker consumption is equal to 0, it means that after QueryContext is created,
235
    // it is found that query already exists in _query_ctx_map, and query mem tracker is not used.
236
    // query mem tracker consumption is not equal to 0 after use, because there is memory consumed
237
    // on query mem tracker, released on other trackers.
238
127k
    std::string mem_tracker_msg;
239
127k
    if (query_mem_tracker()->peak_consumption() != 0) {
240
37
        mem_tracker_msg = fmt::format(
241
37
                "deregister query/load memory tracker, queryId={}, Limit={}, CurrUsed={}, "
242
37
                "PeakUsed={}",
243
37
                print_id(_query_id), PrettyPrinter::print_bytes(query_mem_tracker()->limit()),
244
37
                PrettyPrinter::print_bytes(query_mem_tracker()->consumption()),
245
37
                PrettyPrinter::print_bytes(query_mem_tracker()->peak_consumption()));
246
37
    }
247
127k
    [[maybe_unused]] uint64_t group_id = 0;
248
127k
    if (workload_group()) {
249
46
        group_id = workload_group()->id(); // before remove
250
46
    }
251
252
127k
    _resource_ctx->task_controller()->finish();
253
254
127k
    if (enable_profile()) {
255
0
        _report_query_profile();
256
0
    }
257
258
#ifndef BE_TEST
259
    if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) [[unlikely]] {
260
        try {
261
            ExecEnv::GetInstance()->pipeline_tracer_context()->end_query(_query_id, group_id);
262
        } catch (std::exception& e) {
263
            LOG(WARNING) << "Dump trace log failed bacause " << e.what();
264
        }
265
    }
266
#endif
267
127k
    _runtime_filter_mgr.reset();
268
127k
    _execution_dependency.reset();
269
127k
    _runtime_predicates.clear();
270
127k
    file_scan_range_params_map.clear();
271
127k
    obj_pool.clear();
272
127k
    _merge_controller_handler.reset();
273
274
127k
    if (auto* spill_file_mgr = _exec_env->spill_file_mgr()) {
275
257
        for (auto* data_dir : _spill_data_dirs) {
276
36
            spill_file_mgr->delete_query_spill_directory(print_id(_query_id), data_dir);
277
36
        }
278
257
    }
279
280
127k
    DorisMetrics::instance()->query_ctx_cnt->increment(-1);
281
    // fragment_mgr is nullptr in unittest
282
127k
    if (ExecEnv::GetInstance()->fragment_mgr()) {
283
2
        ExecEnv::GetInstance()->fragment_mgr()->remove_query_context(this->_query_id);
284
2
    }
285
    // the only one msg shows query's end. any other msg should append to it if need.
286
127k
    timespec now;
287
127k
    clock_gettime(CLOCK_MONOTONIC, &now);
288
127k
    int64_t elapsed_ms = (now.tv_sec - _query_arrival_timestamp.tv_sec) * 1000LL +
289
127k
                         (now.tv_nsec - _query_arrival_timestamp.tv_nsec) / 1000000LL;
290
127k
    LOG_INFO("Query {} deconstructed, elapsed_ms: {}, mem_tracker: {}", print_id(this->_query_id),
291
127k
             elapsed_ms, mem_tracker_msg);
292
127k
}
293
294
42
void QueryContext::set_ready_to_execute(Status reason) {
295
42
    set_execution_dependency_ready();
296
42
    _exec_status.update(reason);
297
42
}
298
299
0
void QueryContext::set_ready_to_execute_only() {
300
0
    set_execution_dependency_ready();
301
0
}
302
303
42
void QueryContext::set_execution_dependency_ready() {
304
42
    _execution_dependency->set_ready();
305
42
}
306
307
46
void QueryContext::set_memory_sufficient(bool sufficient) {
308
46
    if (sufficient) {
309
17
        {
310
17
            _memory_sufficient_dependency->set_ready();
311
17
            _resource_ctx->task_controller()->reset_paused_reason();
312
17
        }
313
29
    } else {
314
29
        _memory_sufficient_dependency->block();
315
29
        _resource_ctx->task_controller()->add_paused_count();
316
29
    }
317
46
}
318
319
42
void QueryContext::cancel(Status new_status) {
320
42
    if (!_exec_status.update(new_status)) {
321
0
        return;
322
0
    }
323
    // Tasks should be always runnable.
324
42
    _execution_dependency->set_always_ready();
325
42
    _memory_sufficient_dependency->set_always_ready();
326
42
    if ((new_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>() ||
327
42
         new_status.is<ErrorCode::MEM_ALLOC_FAILED>()) &&
328
42
        _query_options.__isset.dump_heap_profile_when_mem_limit_exceeded &&
329
42
        _query_options.dump_heap_profile_when_mem_limit_exceeded) {
330
        // if query is cancelled because of query mem limit exceeded, dump heap profile
331
        // at the time of cancellation can get the most accurate memory usage for problem analysis
332
0
        auto wg = workload_group();
333
0
        auto log_str = fmt::format(
334
0
                "Query {} canceled because of memory limit exceeded, dumping memory "
335
0
                "detail profiles. wg: {}. {}",
336
0
                print_id(_query_id), wg ? wg->debug_string() : "null",
337
0
                doris::ProcessProfile::instance()->memory_profile()->process_memory_detail_str());
338
0
        LOG_LONG_STRING(INFO, log_str);
339
0
        std::string dot = HeapProfiler::instance()->dump_heap_profile_to_dot();
340
0
        if (!dot.empty()) {
341
0
            dot += "\n-------------------------------------------------------\n";
342
0
            dot += "Copy the text after `digraph` in the above output to "
343
0
                   "http://www.webgraphviz.com to generate a dot graph.\n"
344
0
                   "after start heap profiler, if there is no operation, will print `No nodes "
345
0
                   "to "
346
0
                   "print`."
347
0
                   "If there are many errors: `addr2line: Dwarf Error`,"
348
0
                   "or other FAQ, reference doc: "
349
0
                   "https://doris.apache.org/community/developer-guide/debug-tool/#4-qa\n";
350
0
            auto log_str =
351
0
                    fmt::format("Query {}, dump heap profile to dot: {}", print_id(_query_id), dot);
352
0
            LOG_LONG_STRING(INFO, log_str);
353
0
        }
354
0
    }
355
356
42
    set_ready_to_execute(new_status);
357
358
    // Copy the fragment contexts under the map lock, then cancel them after releasing it. Fragment
359
    // cancellation may take task-level locks and must not run while holding the query map lock.
360
42
    std::vector<std::weak_ptr<PipelineFragmentContext>> ctx_to_cancel;
361
42
    {
362
42
        std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
363
42
        for (auto& entry : _fragment_id_to_pipeline_ctx) {
364
0
            ctx_to_cancel.push_back(entry.second);
365
0
        }
366
42
    }
367
42
    for (auto& f_context : ctx_to_cancel) {
368
0
        if (auto pipeline_ctx = f_context.lock()) {
369
0
            pipeline_ctx->cancel(new_status);
370
0
        }
371
0
    }
372
42
}
373
374
0
void QueryContext::set_load_error_url(std::string error_url) {
375
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
376
0
    _load_error_url = error_url;
377
0
}
378
379
0
std::string QueryContext::get_load_error_url() {
380
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
381
0
    return _load_error_url;
382
0
}
383
384
0
void QueryContext::set_first_error_msg(std::string error_msg) {
385
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
386
0
    _first_error_msg = error_msg;
387
0
}
388
389
0
std::string QueryContext::get_first_error_msg() {
390
0
    std::lock_guard<std::mutex> lock(_error_url_lock);
391
0
    return _first_error_msg;
392
0
}
393
394
0
std::string QueryContext::print_all_pipeline_context() {
395
0
    std::vector<std::weak_ptr<PipelineFragmentContext>> ctx_to_print;
396
0
    fmt::memory_buffer debug_string_buffer;
397
0
    size_t i = 0;
398
0
    {
399
0
        fmt::format_to(debug_string_buffer, "{} pipeline fragment contexts in query {}. \n",
400
0
                       _fragment_id_to_pipeline_ctx.size(), print_id(_query_id));
401
402
0
        {
403
0
            std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
404
0
            for (auto& [f_id, f_context] : _fragment_id_to_pipeline_ctx) {
405
0
                ctx_to_print.push_back(f_context);
406
0
            }
407
0
        }
408
0
        for (auto& f_context : ctx_to_print) {
409
0
            if (auto pipeline_ctx = f_context.lock()) {
410
0
                auto elapsed = pipeline_ctx->elapsed_time() / 1000000000.0;
411
0
                fmt::format_to(debug_string_buffer,
412
0
                               "No.{} (elapse_second={}s, fragment_id={}) : {}\n", i, elapsed,
413
0
                               pipeline_ctx->get_fragment_id(), pipeline_ctx->debug_string());
414
0
                i++;
415
0
            }
416
0
        }
417
0
    }
418
0
    return fmt::to_string(debug_string_buffer);
419
0
}
420
421
28
std::shared_ptr<AIExecutionStatistics> QueryContext::get_ai_statistics(int fragment_id) {
422
28
    std::scoped_lock lock(_ai_statistics_mutex);
423
28
    auto& statistics = _ai_statistics[fragment_id];
424
28
    if (!statistics) {
425
18
        statistics = std::make_shared<AIExecutionStatistics>();
426
18
    }
427
28
    return statistics;
428
28
}
429
430
13
std::shared_ptr<AIExecutionStatistics> QueryContext::find_ai_statistics(int fragment_id) const {
431
13
    std::scoped_lock lock(_ai_statistics_mutex);
432
13
    auto it = _ai_statistics.find(fragment_id);
433
13
    return it == _ai_statistics.end() ? nullptr : it->second;
434
13
}
435
436
void QueryContext::set_pipeline_context(const int fragment_id,
437
0
                                        std::shared_ptr<PipelineFragmentContext> pip_ctx) {
438
0
    std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
439
    // Use insert_or_assign instead of insert to support overwriting old entries
440
    // when recursive CTE recreates PipelineFragmentContext between rounds.
441
0
    _fragment_id_to_pipeline_ctx.insert_or_assign(fragment_id, pip_ctx);
442
0
}
443
444
30
doris::TaskScheduler* QueryContext::get_pipe_exec_scheduler() {
445
30
    if (!_task_scheduler) {
446
0
        throw Exception(Status::InternalError("task_scheduler is null"));
447
0
    }
448
30
    return _task_scheduler;
449
30
}
450
451
45
Status QueryContext::set_workload_group(WorkloadGroupPtr& wg) {
452
45
    _resource_ctx->set_workload_group(wg);
453
    // Should add query first, the workload group will not be deleted,
454
    // then visit workload group's resource
455
    // see task_group_manager::delete_workload_group_by_ids
456
45
    RETURN_IF_ERROR(workload_group()->add_resource_ctx(_query_id, _resource_ctx));
457
458
45
    workload_group()->get_query_scheduler(&_task_scheduler, &_scan_task_scheduler,
459
45
                                          &_remote_scan_task_scheduler);
460
45
    return Status::OK();
461
45
}
462
463
void QueryContext::add_fragment_profile(
464
        int fragment_id, const std::vector<std::shared_ptr<TRuntimeProfileTree>>& pipeline_profiles,
465
4
        std::shared_ptr<TRuntimeProfileTree> load_channel_profile) {
466
4
    if (pipeline_profiles.empty()) {
467
0
        std::string msg = fmt::format("Add pipeline profile failed, query {}, fragment {}",
468
0
                                      print_id(this->_query_id), fragment_id);
469
0
        LOG_ERROR(msg);
470
0
        DCHECK(false) << msg;
471
0
        return;
472
0
    }
473
474
4
#ifndef NDEBUG
475
4
    for (const auto& p : pipeline_profiles) {
476
4
        DCHECK(p != nullptr) << fmt::format("Add pipeline profile failed, query {}, fragment {}",
477
0
                                            print_id(this->_query_id), fragment_id);
478
4
    }
479
4
#endif
480
481
4
    std::lock_guard<std::mutex> l(_profile_mutex);
482
4
    VLOG_ROW << fmt::format(
483
0
            "Query add fragment profile, query {}, fragment {}, pipeline profile count {} ",
484
0
            print_id(this->_query_id), fragment_id, pipeline_profiles.size());
485
486
4
    _profile_map.insert(std::make_pair(fragment_id, pipeline_profiles));
487
488
4
    if (load_channel_profile != nullptr) {
489
0
        _load_channel_profile_map.insert(std::make_pair(fragment_id, load_channel_profile));
490
0
    }
491
4
}
492
493
0
void QueryContext::_report_query_profile() {
494
0
    std::lock_guard<std::mutex> lg(_profile_mutex);
495
496
0
    for (auto& [fragment_id, fragment_profile] : _profile_map) {
497
        // Recursive CTE stages reuse fragment IDs, but the saved profile is the first stage's
498
        // detached snapshot. Refresh only AI counters from their owner, which accumulates all stages.
499
0
        if (auto statistics = find_ai_statistics(fragment_id)) {
500
0
            const int profile_level =
501
0
                    _query_options.__isset.profile_level ? _query_options.profile_level : 2;
502
0
            statistics->update_profile_snapshot(*fragment_profile.front(), profile_level);
503
0
        }
504
0
        std::shared_ptr<TRuntimeProfileTree> load_channel_profile = nullptr;
505
506
0
        if (_load_channel_profile_map.contains(fragment_id)) {
507
0
            load_channel_profile = _load_channel_profile_map[fragment_id];
508
0
        }
509
510
0
        ExecEnv::GetInstance()->runtime_query_statistics_mgr()->register_fragment_profile(
511
0
                _query_id, this->coord_addr, fragment_id, fragment_profile, load_channel_profile);
512
0
    }
513
514
0
    ExecEnv::GetInstance()->runtime_query_statistics_mgr()->trigger_profile_reporting();
515
0
}
516
517
std::unordered_map<int, std::vector<std::shared_ptr<TRuntimeProfileTree>>>
518
0
QueryContext::_collect_realtime_query_profile() {
519
0
    std::unordered_map<int, std::vector<std::shared_ptr<TRuntimeProfileTree>>> res;
520
0
    std::lock_guard<std::mutex> lock(_pipeline_map_write_lock);
521
0
    for (const auto& [fragment_id, fragment_ctx_wptr] : _fragment_id_to_pipeline_ctx) {
522
0
        if (auto fragment_ctx = fragment_ctx_wptr.lock()) {
523
0
            if (fragment_ctx == nullptr) {
524
0
                std::string msg =
525
0
                        fmt::format("PipelineFragmentContext is nullptr, query {} fragment_id: {}",
526
0
                                    print_id(_query_id), fragment_id);
527
0
                LOG_ERROR(msg);
528
0
                DCHECK(false) << msg;
529
0
                continue;
530
0
            }
531
532
0
            auto profile = fragment_ctx->collect_realtime_profile();
533
534
0
            if (profile.empty()) {
535
0
                std::string err_msg = fmt::format(
536
0
                        "Get nothing when collecting profile, query {}, fragment_id: {}",
537
0
                        print_id(_query_id), fragment_id);
538
0
                LOG_ERROR(err_msg);
539
0
                DCHECK(false) << err_msg;
540
0
                continue;
541
0
            }
542
543
0
            res.insert(std::make_pair(fragment_id, profile));
544
0
        }
545
0
    }
546
547
0
    return res;
548
0
}
549
550
0
TReportExecStatusParams QueryContext::get_realtime_exec_status() {
551
0
    TReportExecStatusParams exec_status;
552
553
0
    auto realtime_query_profile = _collect_realtime_query_profile();
554
0
    std::vector<std::shared_ptr<TRuntimeProfileTree>> load_channel_profiles;
555
556
0
    for (auto load_channel_profile : _load_channel_profile_map) {
557
0
        if (load_channel_profile.second != nullptr) {
558
0
            load_channel_profiles.push_back(load_channel_profile.second);
559
0
        }
560
0
    }
561
562
0
    exec_status = RuntimeQueryStatisticsMgr::create_report_exec_status_params(
563
0
            this->_query_id, std::move(realtime_query_profile), std::move(load_channel_profiles),
564
0
            /*is_done=*/false);
565
566
0
    return exec_status;
567
0
}
568
569
Status QueryContext::send_block_to_cte_scan(
570
        const TUniqueId& instance_id, int node_id,
571
0
        const google::protobuf::RepeatedPtrField<doris::PBlock>& pblocks, bool eos) {
572
0
    std::unique_lock<std::mutex> l(_cte_scan_lock);
573
0
    auto it = _cte_scan.find(std::make_pair(instance_id, node_id));
574
0
    if (it == _cte_scan.end()) {
575
0
        return Status::InternalError("RecCTEScan not found for instance {}, node {}",
576
0
                                     print_id(instance_id), node_id);
577
0
    }
578
0
    for (const auto& pblock : pblocks) {
579
0
        RETURN_IF_ERROR(it->second->add_block(pblock));
580
0
    }
581
0
    if (eos) {
582
0
        it->second->set_ready();
583
0
    }
584
0
    return Status::OK();
585
0
}
586
587
void QueryContext::registe_cte_scan(const TUniqueId& instance_id, int node_id,
588
0
                                    RecCTEScanLocalState* scan) {
589
0
    std::unique_lock<std::mutex> l(_cte_scan_lock);
590
0
    auto key = std::make_pair(instance_id, node_id);
591
0
    DCHECK(!_cte_scan.contains(key)) << "Duplicate registe cte scan for instance "
592
0
                                     << print_id(instance_id) << ", node " << node_id;
593
0
    _cte_scan.emplace(key, scan);
594
0
}
595
596
0
void QueryContext::deregiste_cte_scan(const TUniqueId& instance_id, int node_id) {
597
0
    std::lock_guard<std::mutex> l(_cte_scan_lock);
598
0
    auto key = std::make_pair(instance_id, node_id);
599
0
    DCHECK(_cte_scan.contains(key)) << "Duplicate deregiste cte scan for instance "
600
0
                                    << print_id(instance_id) << ", node " << node_id;
601
0
    _cte_scan.erase(key);
602
0
}
603
604
0
Status QueryContext::reset_global_rf(const google::protobuf::RepeatedField<int32_t>& filter_ids) {
605
0
    if (_merge_controller_handler) {
606
0
        return _merge_controller_handler->reset_global_rf(this, filter_ids);
607
0
    }
608
0
    return Status::OK();
609
0
}
610
611
6
void QueryContext::add_total_task_num(int delta) {
612
6
    if (auto* qtc = dynamic_cast<QueryTaskController*>(_resource_ctx->task_controller())) {
613
6
        qtc->add_total_task_num(delta);
614
6
    }
615
6
}
616
617
421
void QueryContext::inc_finished_task_num() {
618
421
    if (auto* qtc = dynamic_cast<QueryTaskController*>(_resource_ctx->task_controller())) {
619
421
        qtc->inc_finished_task_num();
620
421
    }
621
421
}
622
623
} // namespace doris