/root/doris/be/src/runtime/runtime_state.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // This file is copied from |
18 | | // https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/runtime-state.cpp |
19 | | // and modified by Doris |
20 | | |
21 | | #include "runtime/runtime_state.h" |
22 | | |
23 | | #include <fmt/format.h> |
24 | | #include <gen_cpp/PaloInternalService_types.h> |
25 | | #include <gen_cpp/Types_types.h> |
26 | | #include <glog/logging.h> |
27 | | |
28 | | #include <fstream> |
29 | | #include <memory> |
30 | | #include <string> |
31 | | |
32 | | #include "cloud/cloud_storage_engine.h" |
33 | | #include "cloud/config.h" |
34 | | #include "common/config.h" |
35 | | #include "common/logging.h" |
36 | | #include "common/object_pool.h" |
37 | | #include "common/status.h" |
38 | | #include "core/value/vdatetime_value.h" |
39 | | #include "exec/operator/operator.h" |
40 | | #include "exec/pipeline/pipeline_fragment_context.h" |
41 | | #include "exec/pipeline/pipeline_task.h" |
42 | | #include "exec/runtime_filter/runtime_filter_consumer.h" |
43 | | #include "exec/runtime_filter/runtime_filter_mgr.h" |
44 | | #include "exec/runtime_filter/runtime_filter_producer.h" |
45 | | #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" |
46 | | #include "io/fs/s3_file_system.h" |
47 | | #include "load/load_path_mgr.h" |
48 | | #include "runtime/exec_env.h" |
49 | | #include "runtime/fragment_mgr.h" |
50 | | #include "runtime/memory/mem_tracker_limiter.h" |
51 | | #include "runtime/memory/thread_mem_tracker_mgr.h" |
52 | | #include "runtime/query_context.h" |
53 | | #include "runtime/thread_context.h" |
54 | | #include "storage/id_manager.h" |
55 | | #include "storage/storage_engine.h" |
56 | | #include "util/timezone_utils.h" |
57 | | #include "util/uid_util.h" |
58 | | |
59 | | namespace doris { |
60 | | #include "common/compile_check_begin.h" |
61 | | using namespace ErrorCode; |
62 | | |
63 | | RuntimeState::RuntimeState(const TPlanFragmentExecParams& fragment_exec_params, |
64 | | const TQueryOptions& query_options, const TQueryGlobals& query_globals, |
65 | | ExecEnv* exec_env, QueryContext* ctx, |
66 | | const std::shared_ptr<MemTrackerLimiter>& query_mem_tracker) |
67 | 0 | : _profile("Fragment " + print_id(fragment_exec_params.fragment_instance_id)), |
68 | 0 | _load_channel_profile("<unnamed>"), |
69 | 0 | _obj_pool(new ObjectPool()), |
70 | 0 | _unreported_error_idx(0), |
71 | 0 | _query_id(fragment_exec_params.query_id), |
72 | 0 | _per_fragment_instance_idx(0), |
73 | 0 | _num_rows_load_total(0), |
74 | 0 | _num_rows_load_filtered(0), |
75 | 0 | _num_rows_load_unselected(0), |
76 | 0 | _num_print_error_rows(0), |
77 | 0 | _num_bytes_load_total(0), |
78 | 0 | _num_finished_scan_range(0), |
79 | 0 | _query_ctx(ctx) { |
80 | 0 | Status status = |
81 | 0 | init(fragment_exec_params.fragment_instance_id, query_options, query_globals, exec_env); |
82 | 0 | DCHECK(status.ok()); |
83 | 0 | _query_mem_tracker = query_mem_tracker; |
84 | 0 | DCHECK(_query_mem_tracker != nullptr); |
85 | 0 | } |
86 | | |
87 | | RuntimeState::RuntimeState(const TUniqueId& instance_id, const TUniqueId& query_id, |
88 | | int32_t fragment_id, const TQueryOptions& query_options, |
89 | | const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx) |
90 | 81 | : _profile("Fragment " + print_id(instance_id)), |
91 | 81 | _load_channel_profile("<unnamed>"), |
92 | 81 | _obj_pool(new ObjectPool()), |
93 | 81 | _unreported_error_idx(0), |
94 | 81 | _query_id(query_id), |
95 | 81 | _fragment_id(fragment_id), |
96 | 81 | _per_fragment_instance_idx(0), |
97 | 81 | _num_rows_load_total(0), |
98 | 81 | _num_rows_load_filtered(0), |
99 | 81 | _num_rows_load_unselected(0), |
100 | 81 | _num_rows_filtered_in_strict_mode_partial_update(0), |
101 | 81 | _num_print_error_rows(0), |
102 | 81 | _num_bytes_load_total(0), |
103 | 81 | _num_finished_scan_range(0), |
104 | 81 | _query_ctx(ctx) { |
105 | 81 | [[maybe_unused]] auto status = init(instance_id, query_options, query_globals, exec_env); |
106 | 81 | DCHECK(status.ok()); |
107 | 81 | _query_mem_tracker = ctx->query_mem_tracker(); |
108 | 81 | } |
109 | | |
110 | | RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id, |
111 | | const TQueryOptions& query_options, const TQueryGlobals& query_globals, |
112 | | ExecEnv* exec_env, QueryContext* ctx) |
113 | 128 | : _profile("PipelineX " + std::to_string(fragment_id)), |
114 | 128 | _load_channel_profile("<unnamed>"), |
115 | 128 | _obj_pool(new ObjectPool()), |
116 | 128 | _unreported_error_idx(0), |
117 | 128 | _query_id(query_id), |
118 | 128 | _fragment_id(fragment_id), |
119 | 128 | _per_fragment_instance_idx(0), |
120 | 128 | _num_rows_load_total(0), |
121 | 128 | _num_rows_load_filtered(0), |
122 | 128 | _num_rows_load_unselected(0), |
123 | 128 | _num_rows_filtered_in_strict_mode_partial_update(0), |
124 | 128 | _num_print_error_rows(0), |
125 | 128 | _num_bytes_load_total(0), |
126 | 128 | _num_finished_scan_range(0), |
127 | 128 | _query_ctx(ctx) { |
128 | | // TODO: do we really need instance id? |
129 | 128 | Status status = init(TUniqueId(), query_options, query_globals, exec_env); |
130 | 128 | DCHECK(status.ok()); |
131 | 128 | _query_mem_tracker = ctx->query_mem_tracker(); |
132 | 128 | } |
133 | | |
134 | | RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id, |
135 | | const TQueryOptions& query_options, const TQueryGlobals& query_globals, |
136 | | ExecEnv* exec_env, |
137 | | const std::shared_ptr<MemTrackerLimiter>& query_mem_tracker) |
138 | 0 | : _profile("PipelineX " + std::to_string(fragment_id)), |
139 | 0 | _load_channel_profile("<unnamed>"), |
140 | 0 | _obj_pool(new ObjectPool()), |
141 | 0 | _unreported_error_idx(0), |
142 | 0 | _query_id(query_id), |
143 | 0 | _fragment_id(fragment_id), |
144 | 0 | _per_fragment_instance_idx(0), |
145 | 0 | _num_rows_load_total(0), |
146 | 0 | _num_rows_load_filtered(0), |
147 | 0 | _num_rows_load_unselected(0), |
148 | 0 | _num_rows_filtered_in_strict_mode_partial_update(0), |
149 | 0 | _num_print_error_rows(0), |
150 | 0 | _num_bytes_load_total(0), |
151 | 0 | _num_finished_scan_range(0) { |
152 | 0 | Status status = init(TUniqueId(), query_options, query_globals, exec_env); |
153 | 0 | DCHECK(status.ok()); |
154 | 0 | _query_mem_tracker = query_mem_tracker; |
155 | 0 | DCHECK(_query_mem_tracker != nullptr); |
156 | 0 | } |
157 | | |
158 | | RuntimeState::RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) |
159 | 692 | : _profile("<unnamed>"), |
160 | 692 | _load_channel_profile("<unnamed>"), |
161 | 692 | _obj_pool(new ObjectPool()), |
162 | 692 | _unreported_error_idx(0), |
163 | 692 | _per_fragment_instance_idx(0) { |
164 | 692 | Status status = init(TUniqueId(), query_options, query_globals, nullptr); |
165 | 692 | _exec_env = ExecEnv::GetInstance(); |
166 | 692 | init_mem_trackers("<unnamed>"); |
167 | 692 | DCHECK(status.ok()); |
168 | 692 | } |
169 | | |
170 | | RuntimeState::RuntimeState(const TQueryGlobals& query_globals) |
171 | 49.5k | : _profile("<unnamed>"), |
172 | 49.5k | _load_channel_profile("<unnamed>"), |
173 | 49.5k | _obj_pool(new ObjectPool()), |
174 | 49.5k | _unreported_error_idx(0), |
175 | 49.5k | _per_fragment_instance_idx(0) { |
176 | 49.5k | _query_options.batch_size = DEFAULT_BATCH_SIZE; |
177 | 49.5k | if (query_globals.__isset.time_zone && query_globals.__isset.nano_seconds) { |
178 | 49.4k | _timezone = query_globals.time_zone; |
179 | 49.4k | _timestamp_ms = query_globals.timestamp_ms; |
180 | 49.4k | _nano_seconds = query_globals.nano_seconds; |
181 | 49.4k | } else if (query_globals.__isset.time_zone) { |
182 | 0 | _timezone = query_globals.time_zone; |
183 | 0 | _timestamp_ms = query_globals.timestamp_ms; |
184 | 0 | _nano_seconds = 0; |
185 | 153 | } else if (!query_globals.now_string.empty()) { |
186 | 0 | _timezone = TimezoneUtils::default_time_zone; |
187 | 0 | VecDateTimeValue dt; |
188 | 0 | CastParameters params; |
189 | 0 | DORIS_CHECK((CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT, |
190 | 0 | DatelikeTargetType::DATE_TIME>( |
191 | 0 | {query_globals.now_string.c_str(), query_globals.now_string.size()}, dt, nullptr, |
192 | 0 | params))); |
193 | 0 | int64_t timestamp; |
194 | 0 | dt.unix_timestamp(×tamp, _timezone); |
195 | 0 | _timestamp_ms = timestamp * 1000; |
196 | 0 | _nano_seconds = 0; |
197 | 153 | } else { |
198 | | //Unit test may set into here |
199 | 153 | _timezone = TimezoneUtils::default_time_zone; |
200 | 153 | _timestamp_ms = 0; |
201 | 153 | _nano_seconds = 0; |
202 | 153 | } |
203 | 49.5k | TimezoneUtils::find_cctz_time_zone(_timezone, _timezone_obj); |
204 | 49.5k | init_mem_trackers("<unnamed>"); |
205 | 49.5k | } |
206 | | |
207 | | RuntimeState::RuntimeState() |
208 | 130k | : _profile("<unnamed>"), |
209 | 130k | _load_channel_profile("<unnamed>"), |
210 | 130k | _obj_pool(new ObjectPool()), |
211 | 130k | _unreported_error_idx(0), |
212 | 130k | _per_fragment_instance_idx(0) { |
213 | 130k | _query_options.batch_size = DEFAULT_BATCH_SIZE; |
214 | 130k | _query_options.be_exec_version = BeExecVersionManager::get_newest_version(); |
215 | 130k | _timezone = TimezoneUtils::default_time_zone; |
216 | 130k | _timestamp_ms = 0; |
217 | 130k | _nano_seconds = 0; |
218 | 130k | TimezoneUtils::find_cctz_time_zone(_timezone, _timezone_obj); |
219 | 130k | _exec_env = ExecEnv::GetInstance(); |
220 | 130k | init_mem_trackers("<unnamed>"); |
221 | 130k | } |
222 | | |
223 | 181k | RuntimeState::~RuntimeState() { |
224 | 181k | SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_mem_tracker); |
225 | | // close error log file |
226 | 181k | if (_error_log_file != nullptr && _error_log_file->is_open()) { |
227 | 0 | _error_log_file->close(); |
228 | 0 | } |
229 | 181k | _obj_pool->clear(); |
230 | 181k | } |
231 | | |
232 | 2 | const std::set<int>& RuntimeState::get_deregister_runtime_filter() const { |
233 | 2 | return _registered_runtime_filter_ids; |
234 | 2 | } |
235 | | |
236 | 2 | void RuntimeState::merge_register_runtime_filter(const std::set<int>& runtime_filter_ids) { |
237 | 2 | _registered_runtime_filter_ids.insert(runtime_filter_ids.begin(), runtime_filter_ids.end()); |
238 | 2 | } |
239 | | |
240 | | Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, |
241 | 907 | const TQueryGlobals& query_globals, ExecEnv* exec_env) { |
242 | 907 | _fragment_instance_id = fragment_instance_id; |
243 | 907 | _query_options = query_options; |
244 | 907 | _lc_time_names = query_globals.lc_time_names; |
245 | 907 | if (query_globals.__isset.time_zone && query_globals.__isset.nano_seconds) { |
246 | 0 | _timezone = query_globals.time_zone; |
247 | 0 | _timestamp_ms = query_globals.timestamp_ms; |
248 | 0 | _nano_seconds = query_globals.nano_seconds; |
249 | 907 | } else if (query_globals.__isset.time_zone) { |
250 | 0 | _timezone = query_globals.time_zone; |
251 | 0 | _timestamp_ms = query_globals.timestamp_ms; |
252 | 0 | _nano_seconds = 0; |
253 | 907 | } else if (!query_globals.now_string.empty()) { |
254 | 0 | _timezone = TimezoneUtils::default_time_zone; |
255 | 0 | VecDateTimeValue dt; |
256 | 0 | CastParameters params; |
257 | 0 | DORIS_CHECK((CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT, |
258 | 0 | DatelikeTargetType::DATE_TIME>( |
259 | 0 | {query_globals.now_string.c_str(), query_globals.now_string.size()}, dt, nullptr, |
260 | 0 | params))); |
261 | 0 | int64_t timestamp; |
262 | 0 | dt.unix_timestamp(×tamp, _timezone); |
263 | 0 | _timestamp_ms = timestamp * 1000; |
264 | 0 | _nano_seconds = 0; |
265 | 907 | } else { |
266 | | //Unit test may set into here |
267 | 907 | _timezone = TimezoneUtils::default_time_zone; |
268 | 907 | _timestamp_ms = 0; |
269 | 907 | _nano_seconds = 0; |
270 | 907 | } |
271 | 907 | TimezoneUtils::find_cctz_time_zone(_timezone, _timezone_obj); |
272 | | |
273 | 907 | if (query_globals.__isset.load_zero_tolerance) { |
274 | 907 | _load_zero_tolerance = query_globals.load_zero_tolerance; |
275 | 907 | } |
276 | | |
277 | 907 | _exec_env = exec_env; |
278 | | |
279 | 907 | if (_query_options.max_errors <= 0) { |
280 | | // TODO: fix linker error and uncomment this |
281 | | //_query_options.max_errors = config::max_errors; |
282 | 682 | _query_options.max_errors = 100; |
283 | 682 | } |
284 | | |
285 | 907 | if (_query_options.batch_size <= 0) { |
286 | 604 | _query_options.batch_size = DEFAULT_BATCH_SIZE; |
287 | 604 | } |
288 | | |
289 | 907 | _db_name = "insert_stmt"; |
290 | 907 | _import_label = print_id(fragment_instance_id); |
291 | | |
292 | 907 | _profile_level = query_options.__isset.profile_level ? query_options.profile_level : 2; |
293 | | |
294 | 907 | return Status::OK(); |
295 | 907 | } |
296 | | |
297 | 0 | std::weak_ptr<QueryContext> RuntimeState::get_query_ctx_weak() { |
298 | 0 | return _exec_env->fragment_mgr()->get_query_ctx(_query_ctx->query_id()); |
299 | 0 | } |
300 | | |
301 | 181k | void RuntimeState::init_mem_trackers(const std::string& name, const TUniqueId& id) { |
302 | 181k | _query_mem_tracker = MemTrackerLimiter::create_shared( |
303 | 181k | MemTrackerLimiter::Type::OTHER, fmt::format("{}#Id={}", name, print_id(id))); |
304 | 181k | } |
305 | | |
306 | 181 | std::shared_ptr<MemTrackerLimiter> RuntimeState::query_mem_tracker() const { |
307 | 181 | CHECK(_query_mem_tracker != nullptr); |
308 | 181 | return _query_mem_tracker; |
309 | 181 | } |
310 | | |
311 | 11 | WorkloadGroupPtr RuntimeState::workload_group() { |
312 | 11 | return _query_ctx->workload_group(); |
313 | 11 | } |
314 | | |
315 | 0 | bool RuntimeState::log_error(const std::string& error) { |
316 | 0 | std::lock_guard<std::mutex> l(_error_log_lock); |
317 | |
|
318 | 0 | if (_error_log.size() < _query_options.max_errors) { |
319 | 0 | _error_log.push_back(error); |
320 | 0 | return true; |
321 | 0 | } |
322 | | |
323 | 0 | return false; |
324 | 0 | } |
325 | | |
326 | 0 | void RuntimeState::get_unreported_errors(std::vector<std::string>* new_errors) { |
327 | 0 | std::lock_guard<std::mutex> l(_error_log_lock); |
328 | |
|
329 | 0 | if (_unreported_error_idx < _error_log.size()) { |
330 | 0 | new_errors->assign(_error_log.begin() + _unreported_error_idx, _error_log.end()); |
331 | 0 | _unreported_error_idx = (int)_error_log.size(); |
332 | 0 | } |
333 | 0 | } |
334 | | |
335 | 1.10M | bool RuntimeState::is_cancelled() const { |
336 | | // Maybe we should just return _is_cancelled.load() |
337 | 1.10M | return !_exec_status.ok() || (_query_ctx && _query_ctx->is_cancelled()); |
338 | 1.10M | } |
339 | | |
340 | 15 | Status RuntimeState::cancel_reason() const { |
341 | 15 | if (!_exec_status.ok()) { |
342 | 15 | return _exec_status.status(); |
343 | 15 | } |
344 | | |
345 | 0 | if (_query_ctx) { |
346 | 0 | return _query_ctx->exec_status(); |
347 | 0 | } |
348 | | |
349 | 0 | return Status::Cancelled("Query cancelled"); |
350 | 0 | } |
351 | | |
352 | | const int64_t MAX_ERROR_NUM = 50; |
353 | | |
354 | 0 | Status RuntimeState::create_error_log_file() { |
355 | 0 | if (config::save_load_error_log_to_s3 && config::is_cloud_mode()) { |
356 | 0 | _s3_error_fs = std::dynamic_pointer_cast<io::S3FileSystem>( |
357 | 0 | ExecEnv::GetInstance()->storage_engine().to_cloud().latest_fs()); |
358 | 0 | if (_s3_error_fs) { |
359 | 0 | std::stringstream ss; |
360 | | // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html |
361 | | // shorten the path as much as possible to prevent the length of the presigned URL from |
362 | | // exceeding the MySQL error packet size limit |
363 | 0 | ss << "error_log/" << std::hex << _fragment_instance_id.lo; |
364 | 0 | _s3_error_log_file_path = ss.str(); |
365 | 0 | } |
366 | 0 | } |
367 | |
|
368 | 0 | static_cast<void>(_exec_env->load_path_mgr()->get_load_error_file_name( |
369 | 0 | _db_name, _import_label, _fragment_instance_id, &_error_log_file_path)); |
370 | 0 | std::string error_log_absolute_path = |
371 | 0 | _exec_env->load_path_mgr()->get_load_error_absolute_path(_error_log_file_path); |
372 | 0 | _error_log_file = std::make_unique<std::ofstream>(error_log_absolute_path, std::ifstream::out); |
373 | 0 | if (!_error_log_file->is_open()) { |
374 | 0 | std::stringstream error_msg; |
375 | 0 | error_msg << "Fail to open error file: [" << _error_log_file_path << "]."; |
376 | 0 | LOG(WARNING) << error_msg.str(); |
377 | 0 | return Status::InternalError(error_msg.str()); |
378 | 0 | } |
379 | 0 | LOG(INFO) << "create error log file: " << _error_log_file_path |
380 | 0 | << ", query id: " << print_id(_query_id) |
381 | 0 | << ", fragment instance id: " << print_id(_fragment_instance_id); |
382 | |
|
383 | 0 | return Status::OK(); |
384 | 0 | } |
385 | | |
386 | | Status RuntimeState::append_error_msg_to_file(std::function<std::string()> line, |
387 | 0 | std::function<std::string()> error_msg) { |
388 | 0 | if (query_type() != TQueryType::LOAD) { |
389 | 0 | return Status::OK(); |
390 | 0 | } |
391 | | // If file haven't been opened, open it here |
392 | 0 | if (_error_log_file == nullptr) { |
393 | 0 | Status status = create_error_log_file(); |
394 | 0 | if (!status.ok()) { |
395 | 0 | LOG(WARNING) << "Create error file log failed. because: " << status; |
396 | 0 | if (_error_log_file != nullptr) { |
397 | 0 | _error_log_file->close(); |
398 | 0 | } |
399 | 0 | return status; |
400 | 0 | } |
401 | | // record the first error message if the file is just created |
402 | 0 | _first_error_msg = error_msg() + ". Src line: " + line(); |
403 | 0 | LOG(INFO) << "The first error message: " << _first_error_msg; |
404 | 0 | } |
405 | | // If num of printed error row exceeds the limit, don't add error messages to error log file any more |
406 | 0 | if (_num_print_error_rows.fetch_add(1, std::memory_order_relaxed) > MAX_ERROR_NUM) { |
407 | | // if _load_zero_tolerance, return Error to stop the load process immediately. |
408 | 0 | if (_load_zero_tolerance) { |
409 | 0 | return Status::DataQualityError( |
410 | 0 | "Encountered unqualified data, stop processing. Please check if the source " |
411 | 0 | "data matches the schema, and consider disabling strict mode or increasing " |
412 | 0 | "max_filter_ratio."); |
413 | 0 | } |
414 | 0 | return Status::OK(); |
415 | 0 | } |
416 | | |
417 | 0 | fmt::memory_buffer out; |
418 | | // Note: export reason first in case src line too long and be truncated. |
419 | 0 | fmt::format_to(out, "Reason: {}. src line [{}]; ", error_msg(), line()); |
420 | |
|
421 | 0 | size_t error_row_size = out.size(); |
422 | 0 | if (error_row_size > 0) { |
423 | 0 | if (error_row_size > config::load_error_log_limit_bytes) { |
424 | 0 | fmt::memory_buffer limit_byte_out; |
425 | 0 | limit_byte_out.append(out.data(), out.data() + config::load_error_log_limit_bytes); |
426 | 0 | (*_error_log_file) << fmt::to_string(limit_byte_out) + "error log is too long" |
427 | 0 | << std::endl; |
428 | 0 | } else { |
429 | 0 | (*_error_log_file) << fmt::to_string(out) << std::endl; |
430 | 0 | } |
431 | 0 | } |
432 | |
|
433 | 0 | return Status::OK(); |
434 | 0 | } |
435 | | |
436 | 1 | std::string RuntimeState::get_error_log_file_path() { |
437 | 1 | DBUG_EXECUTE_IF("RuntimeState::get_error_log_file_path.block", { |
438 | 1 | if (!_error_log_file_path.empty()) { |
439 | 1 | std::this_thread::sleep_for(std::chrono::seconds(1)); |
440 | 1 | } |
441 | 1 | }); |
442 | 1 | std::lock_guard<std::mutex> l(_s3_error_log_file_lock); |
443 | 1 | if (_s3_error_fs && _error_log_file && _error_log_file->is_open()) { |
444 | | // close error log file |
445 | 0 | _error_log_file->close(); |
446 | 0 | std::string error_log_absolute_path = |
447 | 0 | _exec_env->load_path_mgr()->get_load_error_absolute_path(_error_log_file_path); |
448 | | // upload error log file to s3 |
449 | 0 | Status st = _s3_error_fs->upload(error_log_absolute_path, _s3_error_log_file_path); |
450 | 0 | if (!st.ok()) { |
451 | | // upload failed and return local error log file path |
452 | 0 | LOG(WARNING) << "Fail to upload error file to s3, error_log_file_path=" |
453 | 0 | << _error_log_file_path << ", error=" << st; |
454 | 0 | return _error_log_file_path; |
455 | 0 | } |
456 | | // expiration must be less than a week (in seconds) for presigned url |
457 | 0 | static const unsigned EXPIRATION_SECONDS = 7 * 24 * 60 * 60 - 1; |
458 | | // Use public or private endpoint based on configuration |
459 | 0 | _error_log_file_path = |
460 | 0 | _s3_error_fs->generate_presigned_url(_s3_error_log_file_path, EXPIRATION_SECONDS, |
461 | 0 | config::use_public_endpoint_for_error_log); |
462 | 0 | } |
463 | 1 | return _error_log_file_path; |
464 | 1 | } |
465 | | |
466 | 72.4k | void RuntimeState::resize_op_id_to_local_state(int operator_size) { |
467 | 72.4k | _op_id_to_local_state.resize(-operator_size); |
468 | 72.4k | } |
469 | | |
470 | | void RuntimeState::emplace_local_state(int id, |
471 | 24.3k | std::unique_ptr<doris::PipelineXLocalStateBase> state) { |
472 | 24.3k | id = -id; |
473 | 24.3k | DCHECK_LT(id, _op_id_to_local_state.size()) |
474 | 0 | << state->parent()->get_name() << " node id = " << state->parent()->node_id(); |
475 | 24.3k | DCHECK(!_op_id_to_local_state[id]); |
476 | 24.3k | _op_id_to_local_state[id] = std::move(state); |
477 | 24.3k | } |
478 | | |
479 | 3.15M | doris::PipelineXLocalStateBase* RuntimeState::get_local_state(int id) { |
480 | 3.15M | id = -id; |
481 | 3.15M | return _op_id_to_local_state[id].get(); |
482 | 3.15M | } |
483 | | |
484 | 24.1k | Result<RuntimeState::LocalState*> RuntimeState::get_local_state_result(int id) { |
485 | 24.1k | id = -id; |
486 | 24.1k | if (id >= _op_id_to_local_state.size()) { |
487 | 1 | return ResultError(Status::InternalError("get_local_state out of range size:{} , id:{}", |
488 | 1 | _op_id_to_local_state.size(), id)); |
489 | 1 | } |
490 | 24.1k | if (!_op_id_to_local_state[id]) { |
491 | 0 | return ResultError(Status::InternalError("get_local_state id:{} is null", id)); |
492 | 0 | } |
493 | 24.1k | return _op_id_to_local_state[id].get(); |
494 | 24.1k | }; |
495 | | |
496 | | void RuntimeState::emplace_sink_local_state( |
497 | 72.3k | int id, std::unique_ptr<doris::PipelineXSinkLocalStateBase> state) { |
498 | 72.3k | DCHECK(!_sink_local_state) << " id=" << id << " state: " << state->debug_string(0); |
499 | 72.3k | _sink_local_state = std::move(state); |
500 | 72.3k | } |
501 | | |
502 | 446k | doris::PipelineXSinkLocalStateBase* RuntimeState::get_sink_local_state() { |
503 | 446k | return _sink_local_state.get(); |
504 | 446k | } |
505 | | |
506 | 1.57M | Result<RuntimeState::SinkLocalState*> RuntimeState::get_sink_local_state_result() { |
507 | 1.57M | if (!_sink_local_state) { |
508 | 0 | return ResultError(Status::InternalError("_op_id_to_sink_local_state not exist")); |
509 | 0 | } |
510 | 1.57M | return _sink_local_state.get(); |
511 | 1.57M | } |
512 | | |
513 | 0 | bool RuntimeState::enable_page_cache() const { |
514 | 0 | return !config::disable_storage_page_cache && |
515 | 0 | (_query_options.__isset.enable_page_cache && _query_options.enable_page_cache); |
516 | 0 | } |
517 | | |
518 | 63 | RuntimeFilterMgr* RuntimeState::global_runtime_filter_mgr() { |
519 | 63 | return _query_ctx->runtime_filter_mgr(); |
520 | 63 | } |
521 | | |
522 | | Status RuntimeState::register_producer_runtime_filter( |
523 | 30 | const TRuntimeFilterDesc& desc, std::shared_ptr<RuntimeFilterProducer>* producer_filter) { |
524 | 30 | _registered_runtime_filter_ids.insert(desc.filter_id); |
525 | | // Producers are created by local runtime filter mgr and shared by global runtime filter manager. |
526 | | // When RF is published, consumers in both global and local RF mgr will be found. |
527 | 30 | RETURN_IF_ERROR(local_runtime_filter_mgr()->register_producer_filter(_query_ctx, desc, |
528 | 30 | producer_filter)); |
529 | | // Stamp the producer with the current recursive CTE stage so that outgoing merge RPCs |
530 | | // carry the correct round number and stale messages from old rounds are discarded. |
531 | | // PFC must still be alive: this runs inside a pipeline task, so the execution context |
532 | | // cannot have expired yet. |
533 | | // In unit-test scenarios the task execution context is never set (no PipelineFragmentContext |
534 | | // exists), so skip the stage stamping — the default stage (0) is correct. |
535 | 30 | if (task_execution_context_inited()) { |
536 | 2 | auto pfc = std::static_pointer_cast<PipelineFragmentContext>( |
537 | 2 | get_task_execution_context().lock()); |
538 | 2 | DORIS_CHECK(pfc); |
539 | 2 | (*producer_filter)->set_stage(pfc->rec_cte_stage()); |
540 | 2 | } |
541 | 30 | RETURN_IF_ERROR(global_runtime_filter_mgr()->register_local_merge_producer_filter( |
542 | 30 | _query_ctx, desc, *producer_filter)); |
543 | 30 | return Status::OK(); |
544 | 30 | } |
545 | | |
546 | | Status RuntimeState::register_consumer_runtime_filter( |
547 | | const TRuntimeFilterDesc& desc, bool need_local_merge, int node_id, |
548 | 8 | std::shared_ptr<RuntimeFilterConsumer>* consumer_filter) { |
549 | 8 | _registered_runtime_filter_ids.insert(desc.filter_id); |
550 | 8 | bool need_merge = desc.has_remote_targets || need_local_merge; |
551 | 8 | RuntimeFilterMgr* mgr = need_merge ? global_runtime_filter_mgr() : local_runtime_filter_mgr(); |
552 | 8 | RETURN_IF_ERROR(mgr->register_consumer_filter(this, desc, node_id, consumer_filter)); |
553 | | // Stamp the consumer with the current recursive CTE stage so that incoming publish RPCs |
554 | | // from old rounds are detected and discarded. |
555 | | // PFC must still be alive: this runs inside a pipeline task, so the execution context |
556 | | // cannot have expired yet. |
557 | | // In unit-test scenarios the task execution context is never set (no PipelineFragmentContext |
558 | | // exists), so skip the stage stamping — the default stage (0) is correct. |
559 | 8 | if (task_execution_context_inited()) { |
560 | 0 | auto pfc = std::static_pointer_cast<PipelineFragmentContext>( |
561 | 0 | get_task_execution_context().lock()); |
562 | 0 | DORIS_CHECK(pfc); |
563 | 0 | (*consumer_filter)->set_stage(pfc->rec_cte_stage()); |
564 | 0 | } |
565 | 8 | return Status::OK(); |
566 | 8 | } |
567 | | |
568 | 74 | bool RuntimeState::is_nereids() const { |
569 | 74 | return _query_ctx->is_nereids(); |
570 | 74 | } |
571 | | |
572 | 0 | std::vector<std::shared_ptr<RuntimeProfile>> RuntimeState::pipeline_id_to_profile() { |
573 | 0 | std::shared_lock lc(_pipeline_profile_lock); |
574 | 0 | return _pipeline_id_to_profile; |
575 | 0 | } |
576 | | |
577 | | std::vector<std::shared_ptr<RuntimeProfile>> RuntimeState::build_pipeline_profile( |
578 | 0 | std::size_t pipeline_size) { |
579 | 0 | std::unique_lock lc(_pipeline_profile_lock); |
580 | 0 | if (!_pipeline_id_to_profile.empty()) { |
581 | 0 | return _pipeline_id_to_profile; |
582 | 0 | } |
583 | 0 | _pipeline_id_to_profile.resize(pipeline_size); |
584 | 0 | { |
585 | 0 | size_t pip_idx = 0; |
586 | 0 | for (auto& pipeline_profile : _pipeline_id_to_profile) { |
587 | 0 | pipeline_profile = |
588 | 0 | std::make_shared<RuntimeProfile>("Pipeline : " + std::to_string(pip_idx)); |
589 | 0 | pip_idx++; |
590 | 0 | } |
591 | 0 | } |
592 | 0 | return _pipeline_id_to_profile; |
593 | 0 | } |
594 | | |
595 | 1.02M | bool RuntimeState::low_memory_mode() const { |
596 | 1.02M | #ifdef BE_TEST |
597 | 1.02M | if (!_query_ctx) { |
598 | 0 | return false; |
599 | 0 | } |
600 | 1.02M | #endif |
601 | 1.02M | return _query_ctx->low_memory_mode(); |
602 | 1.02M | } |
603 | | |
604 | 0 | void RuntimeState::set_id_file_map() { |
605 | 0 | _id_file_map = _exec_env->get_id_manager()->add_id_file_map(_query_id, execution_timeout()); |
606 | 0 | } |
607 | | #include "common/compile_check_end.h" |
608 | | } // end namespace doris |