Coverage Report

Created: 2025-07-23 12:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/common/config.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include <fmt/core.h>
19
#include <gflags/gflags.h>
20
#include <stdint.h>
21
22
#include <algorithm>
23
#include <cctype>
24
// IWYU pragma: no_include <bthread/errno.h>
25
#include <lz4/lz4hc.h>
26
27
#include <cerrno> // IWYU pragma: keep
28
#include <cstdlib>
29
#include <cstring>
30
#include <fstream> // IWYU pragma: keep
31
#include <functional>
32
#include <iostream>
33
#include <map>
34
#include <memory>
35
#include <mutex>
36
#include <random>
37
#include <string>
38
#include <utility>
39
#include <vector>
40
41
#include "cloud/config.h"
42
#include "common/config.h"
43
#include "common/logging.h"
44
#include "common/status.h"
45
#include "io/fs/file_writer.h"
46
#include "io/fs/local_file_system.h"
47
#include "util/cpu_info.h"
48
49
namespace doris::config {
50
51
// Dir of custom config file
52
DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf");
53
54
// Dir of jdbc drivers
55
DEFINE_String(jdbc_drivers_dir, "${DORIS_HOME}/plugins/jdbc_drivers");
56
57
// cluster id
58
DEFINE_Int32(cluster_id, "-1");
59
// port on which BackendService is exported
60
DEFINE_Int32(be_port, "9060");
61
62
// port for brpc
63
DEFINE_Int32(brpc_port, "8060");
64
65
DEFINE_Int32(arrow_flight_sql_port, "8050");
66
67
// If the external client cannot directly access priority_networks, set public_host to be accessible
68
// to external client.
69
// There are usually two usage scenarios:
70
// 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network.
71
// However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be
72
// randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx.
73
// 2. if priority_networks is an internal network IP, and BE node has its own independent external IP,
74
// but Doris currently does not support modifying priority_networks, setting public_host to the real external IP.
75
DEFINE_mString(public_host, "");
76
77
// If the BE node is connected to the external network through a reverse proxy like Nginx
78
// and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy
79
// `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example:
80
// upstream arrowflight {
81
//    server 10.16.10.8:8069;
82
//    server 10.16.10.8:8068;
83
//}
84
// server {
85
//    listen 8167 http2;
86
//    listen [::]:8167 http2;
87
//    server_name doris.arrowflight.com;
88
// }
89
DEFINE_Int32(arrow_flight_sql_proxy_port, "-1");
90
91
// the number of bthreads for brpc, the default value is set to -1,
92
// which means the number of bthreads is #cpu-cores
93
DEFINE_Int32(brpc_num_threads, "256");
94
// the time of brpc server keep idle connection, setting this value too small may cause rpc between backends to fail,
95
// the default value is set to -1, which means never close idle connection.
96
DEFINE_Int32(brpc_idle_timeout_sec, "-1");
97
98
// Declare a selection strategy for those servers have many ips.
99
// Note that there should at most one ip match this list.
100
// this is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
101
// If no ip match this rule, will choose one randomly.
102
DEFINE_String(priority_networks, "");
103
104
// memory mode
105
// performance or compact
106
DEFINE_String(memory_mode, "moderate");
107
108
DEFINE_mBool(enable_use_cgroup_memory_info, "true");
109
110
// process memory limit specified as number of bytes
111
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
112
// or percentage of the physical memory ('<int>%').
113
// defaults to bytes if no unit is given"
114
// must larger than 0. and if larger than physical memory size,
115
// it will be set to physical memory size.
116
DEFINE_String(mem_limit, "90%");
117
118
// Soft memory limit as a fraction of hard memory limit.
119
DEFINE_Double(soft_mem_limit_frac, "0.9");
120
121
// Cache capacity reduce mem limit as a fraction of soft mem limit.
122
DEFINE_mDouble(cache_capacity_reduce_mem_limit_frac, "0.7");
123
124
// Schema change memory limit as a fraction of soft memory limit.
125
DEFINE_Double(schema_change_mem_limit_frac, "0.6");
126
127
// Many modern allocators (for example, tcmalloc) do not do a mremap for
128
// realloc, even in case of large enough chunks of memory. Although this allows
129
// you to increase performance and reduce memory consumption during realloc.
130
// To fix this, we do mremap manually if the chunk of memory is large enough.
131
//
132
// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
133
// space is very slow, especially in the case of a large number of threads. We
134
// expect that the set of operations mmap/something to do/mremap can only be
135
// performed about 1000 times per second.
136
//
137
// P.S. This is also required, because tcmalloc can not allocate a chunk of
138
// memory greater than 16 GB.
139
DEFINE_mInt64(mmap_threshold, "134217728"); // bytes
140
141
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
142
// Increase can reduce the number of hash table resize, but may waste more memory.
143
DEFINE_mInt32(hash_table_double_grow_degree, "31");
144
145
DEFINE_mInt32(max_fill_rate, "2");
146
147
DEFINE_mInt32(double_resize_threshold, "23");
148
149
// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default -1.
150
// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%), which is 3.2G on a 64G machine.
151
// Turn up max. more memory buffers will be reserved for Memory GC.
152
// Turn down max. will use as much memory as possible.
153
// note that: `max_` prefix should be removed, but keep it for compatibility.
154
DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "-1");
155
156
DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
157
158
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
159
DEFINE_mString(process_minor_gc_size, "5%");
160
DEFINE_mString(process_full_gc_size, "10%");
161
162
// gc will release cache, cancel task, and task will wait for gc to release memory,
163
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
164
DEFINE_mBool(disable_memory_gc, "false");
165
166
// for the query being canceled,
167
// if (current time - cancel start time) < revoke_memory_max_tolerance_ms, the query memory is counted in `freed_memory`,
168
// and the query memory is expected to be released soon.
169
// if > revoke_memory_max_tolerance_ms, the query memory will not be counted in `freed_memory`,
170
// and the query may be blocked during the cancel process. skip this query and continue to cancel other queries.
171
DEFINE_mInt64(revoke_memory_max_tolerance_ms, "3000");
172
173
DEFINE_mBool(enable_stacktrace, "true");
174
175
DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes, "2147483647"); // 2GB -1
176
177
DEFINE_mInt64(crash_in_alloc_large_memory_bytes, "-1");
178
179
// The actual meaning of this parameter is `debug_memory`.
180
// 1. crash in memory tracker inaccurate, if memory tracker value is inaccurate, BE will crash.
181
//    usually used in test environments, default value is false.
182
// 2. print more memory logs.
183
DEFINE_mBool(crash_in_memory_tracker_inaccurate, "false");
184
185
// default is true. if any memory tracking in Orphan mem tracker will report error.
186
// !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time.
187
// allocator free memory not need to check, because when the thread memory tracker label is Orphan,
188
// use the tracker saved in Allocator.
189
DEFINE_mBool(enable_memory_orphan_check, "true");
190
191
// The maximum time a thread waits for full GC. Currently only query will wait for full gc.
192
DEFINE_mInt32(thread_wait_gc_max_milliseconds, "1000");
193
194
DEFINE_mInt64(pre_serialize_keys_limit_bytes, "16777216");
195
196
// the port heartbeat service used
197
DEFINE_Int32(heartbeat_service_port, "9050");
198
// the count of heart beat service
199
DEFINE_Int32(heartbeat_service_thread_count, "1");
200
// the count of thread to create table
201
DEFINE_Int32(create_tablet_worker_count, "3");
202
// the count of thread to drop table
203
DEFINE_Int32(drop_tablet_worker_count, "3");
204
// the count of thread to batch load
205
DEFINE_Int32(push_worker_count_normal_priority, "3");
206
// the count of thread to high priority batch load
207
DEFINE_Int32(push_worker_count_high_priority, "3");
208
// the count of thread to publish version
209
DEFINE_Int32(publish_version_worker_count, "8");
210
// the count of tablet thread to publish version
211
DEFINE_Int32(tablet_publish_txn_max_thread, "32");
212
// the timeout of EnginPublishVersionTask
213
DEFINE_Int32(publish_version_task_timeout_s, "8");
214
// the count of thread to calc delete bitmap
215
DEFINE_Int32(calc_delete_bitmap_max_thread, "32");
216
// the count of thread to calc delete bitmap worker, only used for cloud
217
DEFINE_Int32(calc_delete_bitmap_worker_count, "8");
218
// the count of thread to calc tablet delete bitmap task, only used for cloud
219
DEFINE_Int32(calc_tablet_delete_bitmap_task_max_thread, "32");
220
// the count of thread to clear transaction task
221
DEFINE_Int32(clear_transaction_task_worker_count, "1");
222
// the count of thread to delete
223
DEFINE_Int32(delete_worker_count, "3");
224
// the count of thread to alter table
225
DEFINE_Int32(alter_tablet_worker_count, "3");
226
// the count of thread to alter index
227
DEFINE_Int32(alter_index_worker_count, "3");
228
// the count of thread to clone
229
DEFINE_Int32(clone_worker_count, "3");
230
// the count of thread to clone
231
DEFINE_Int32(storage_medium_migrate_count, "1");
232
// the count of thread to check consistency
233
DEFINE_Int32(check_consistency_worker_count, "1");
234
// the count of thread to upload
235
DEFINE_Int32(upload_worker_count, "1");
236
// the count of thread to download
237
DEFINE_Int32(download_worker_count, "1");
238
DEFINE_Int32(num_query_ctx_map_partitions, "128");
239
// the count of thread to make snapshot
240
DEFINE_Int32(make_snapshot_worker_count, "5");
241
// the count of thread to release snapshot
242
DEFINE_Int32(release_snapshot_worker_count, "5");
243
// report random wait a little time to avoid FE receiving multiple be reports at the same time.
244
// do not set it to false for production environment
245
DEFINE_mBool(report_random_wait, "true");
246
// the interval time(seconds) for agent report tasks signature to FE
247
DEFINE_mInt32(report_task_interval_seconds, "10");
248
// the interval time(seconds) for agent report disk state to FE
249
DEFINE_mInt32(report_disk_state_interval_seconds, "60");
250
// the interval time(seconds) for agent report olap table to FE
251
DEFINE_mInt32(report_tablet_interval_seconds, "60");
252
// the max download speed(KB/s)
253
DEFINE_mInt32(max_download_speed_kbps, "50000");
254
// download low speed limit(KB/s)
255
DEFINE_mInt32(download_low_speed_limit_kbps, "50");
256
// download low speed time(seconds)
257
DEFINE_mInt32(download_low_speed_time, "300");
258
// whether to download small files in batch
259
DEFINE_mBool(enable_batch_download, "true");
260
// whether to check md5sum when download
261
DEFINE_mBool(enable_download_md5sum_check, "false");
262
// download binlog meta timeout, default 30s
263
DEFINE_mInt32(download_binlog_meta_timeout_ms, "30000");
264
// the interval time(seconds) for agent report index policy to FE
265
DEFINE_mInt32(report_index_policy_interval_seconds, "10");
266
267
DEFINE_String(sys_log_dir, "");
268
DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");
269
// INFO, WARNING, ERROR, FATAL
270
DEFINE_mString(sys_log_level, "INFO");
271
// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
272
DEFINE_String(sys_log_roll_mode, "SIZE-MB-1024");
273
// log roll num
274
DEFINE_Int32(sys_log_roll_num, "10");
275
// verbose log
276
DEFINE_Strings(sys_log_verbose_modules, "");
277
// verbose log level
278
DEFINE_Int32(sys_log_verbose_level, "10");
279
// verbose log FLAGS_v
280
DEFINE_Int32(sys_log_verbose_flags_v, "-1");
281
// log buffer level
282
DEFINE_String(log_buffer_level, "");
283
// log enable custom date time format
284
DEFINE_Bool(sys_log_enable_custom_date_time_format, "false");
285
// log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time)
286
DEFINE_String(sys_log_custom_date_time_format, "%Y-%m-%d %H:%M:%S");
287
// log custom date time milliseconds format (fmt::format)
288
DEFINE_String(sys_log_custom_date_time_ms_format, ",{:03d}");
289
290
// number of threads available to serve backend execution requests
291
DEFINE_Int32(be_service_threads, "64");
292
293
// The pipeline task has a high concurrency, therefore reducing its report frequency
294
DEFINE_mInt32(pipeline_status_report_interval, "10");
295
DEFINE_mInt32(pipeline_task_exec_time_slice, "100");
296
297
// task executor min concurrency per task
298
DEFINE_Int32(task_executor_min_concurrency_per_task, "1");
299
// task executor max concurrency per task
300
DEFINE_Int32(task_executor_max_concurrency_per_task, "-1");
301
DEFINE_Validator(task_executor_max_concurrency_per_task, [](const int config) -> bool {
302
    if (config == -1) {
303
        task_executor_max_concurrency_per_task = std::numeric_limits<int>::max();
304
    }
305
    return true;
306
});
307
// task task executor inital split max concurrency per task, later concurrency may be adjusted dynamically
308
DEFINE_Int32(task_executor_initial_max_concurrency_per_task, "-1");
309
DEFINE_Validator(task_executor_initial_max_concurrency_per_task, [](const int config) -> bool {
310
    if (config == -1) {
311
        CpuInfo::init();
312
        task_executor_initial_max_concurrency_per_task = std::max(48, CpuInfo::num_cores() * 2);
313
    }
314
    return true;
315
});
316
// Enable task executor in internal table scan.
317
DEFINE_Bool(enable_task_executor_in_internal_table, "true");
318
// Enable task executor in external table scan.
319
DEFINE_Bool(enable_task_executor_in_external_table, "true");
320
321
// number of scanner thread pool size for olap table
322
// and the min thread num of remote scanner thread pool
323
DEFINE_Int32(doris_scanner_thread_pool_thread_num, "-1");
324
DEFINE_Validator(doris_scanner_thread_pool_thread_num, [](const int config) -> bool {
325
    if (config == -1) {
326
        CpuInfo::init();
327
        doris_scanner_thread_pool_thread_num = std::max(48, CpuInfo::num_cores() * 2);
328
    }
329
    return true;
330
});
331
DEFINE_Int32(doris_scanner_min_thread_pool_thread_num, "8");
332
DEFINE_Int32(remote_split_source_batch_size, "1000");
333
DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1");
334
// number of olap scanner thread pool queue size
335
DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400");
336
// default thrift client connect timeout(in seconds)
337
DEFINE_mInt32(thrift_connect_timeout_seconds, "3");
338
339
// default thrift client retry interval (in milliseconds)
340
DEFINE_mInt64(thrift_client_retry_interval_ms, "1000");
341
// max message size of thrift request
342
// default: 100 * 1024 * 1024
343
DEFINE_mInt64(thrift_max_message_size, "104857600");
344
// max bytes number for single scan range, used in segmentv2
345
DEFINE_mInt32(doris_scan_range_max_mb, "1024");
346
// single read execute fragment row number
347
DEFINE_mInt32(doris_scanner_row_num, "16384");
348
// single read execute fragment row bytes
349
DEFINE_mInt32(doris_scanner_row_bytes, "10485760");
350
// single read execute fragment max run time millseconds
351
DEFINE_mInt32(doris_scanner_max_run_time_ms, "1000");
352
// (Advanced) Maximum size of per-query receive-side buffer
353
DEFINE_mInt32(exchg_node_buffer_size_bytes, "20485760");
354
DEFINE_mInt32(exchg_buffer_queue_capacity_factor, "64");
355
356
// memory_limitation_per_thread_for_schema_change_bytes unit bytes
357
DEFINE_mInt64(memory_limitation_per_thread_for_schema_change_bytes, "2147483648");
358
359
DEFINE_mInt32(cache_prune_interval_sec, "10");
360
DEFINE_mInt32(cache_periodic_prune_stale_sweep_sec, "60");
361
// the clean interval of tablet lookup cache
362
DEFINE_mInt32(tablet_lookup_cache_stale_sweep_time_sec, "30");
363
DEFINE_mInt32(point_query_row_cache_stale_sweep_time_sec, "300");
364
DEFINE_mInt32(disk_stat_monitor_interval, "5");
365
DEFINE_mInt32(unused_rowset_monitor_interval, "30");
366
DEFINE_mInt32(quering_rowsets_evict_interval, "30");
367
DEFINE_String(storage_root_path, "${DORIS_HOME}/storage");
368
DEFINE_mString(broken_storage_path, "");
369
370
// Config is used to check incompatible old format hdr_ format
371
// whether doris uses strict way. When config is true, process will log fatal
372
// and exit. When config is false, process will only log warning.
373
DEFINE_Bool(storage_strict_check_incompatible_old_format, "true");
374
375
// BE process will exit if the percentage of error disk reach this value.
376
DEFINE_mInt32(max_percentage_of_error_disk, "100");
377
DEFINE_mInt32(default_num_rows_per_column_file_block, "1024");
378
// pending data policy
379
DEFINE_mInt32(pending_data_expire_time_sec, "1800");
380
// inc_rowset snapshot rs sweep time interval
381
DEFINE_mInt32(tablet_rowset_stale_sweep_time_sec, "600");
382
// tablet stale rowset sweep by threshold size
383
DEFINE_Bool(tablet_rowset_stale_sweep_by_size, "false");
384
DEFINE_mInt32(tablet_rowset_stale_sweep_threshold_size, "100");
385
// garbage sweep policy
386
DEFINE_Int32(max_garbage_sweep_interval, "3600");
387
DEFINE_Int32(min_garbage_sweep_interval, "180");
388
DEFINE_mInt32(garbage_sweep_batch_size, "100");
389
DEFINE_mInt32(snapshot_expire_time_sec, "172800");
390
// It is only a recommended value. When the disk space is insufficient,
391
// the file storage period under trash dose not have to comply with this parameter.
392
DEFINE_mInt32(trash_file_expire_time_sec, "0");
393
// minimum file descriptor number
394
// modify them upon necessity
395
DEFINE_Int32(min_file_descriptor_number, "60000");
396
DEFINE_mBool(disable_segment_cache, "false");
397
DEFINE_String(row_cache_mem_limit, "20%");
398
399
// Cache for storage page size
400
DEFINE_String(storage_page_cache_limit, "20%");
401
// Shard size for page cache, the value must be power of two.
402
// It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions.
403
DEFINE_Int32(storage_page_cache_shard_size, "256");
404
// Percentage for index page cache
405
// all storage page cache will be divided into data_page_cache and index_page_cache
406
DEFINE_Int32(index_page_cache_percentage, "10");
407
// whether to disable page cache feature in storage
408
DEFINE_mBool(disable_storage_page_cache, "false");
409
// whether to disable row cache feature in storage
410
DEFINE_mBool(disable_storage_row_cache, "true");
411
// whether to disable pk page cache feature in storage
412
DEFINE_Bool(disable_pk_storage_page_cache, "false");
413
414
// Cache for mow primary key storage page size
415
DEFINE_String(pk_storage_page_cache_limit, "10%");
416
// data page size for primary key index
417
DEFINE_Int32(primary_key_data_page_size, "32768");
418
419
DEFINE_mInt32(data_page_cache_stale_sweep_time_sec, "300");
420
DEFINE_mInt32(index_page_cache_stale_sweep_time_sec, "600");
421
DEFINE_mInt32(pk_index_page_cache_stale_sweep_time_sec, "600");
422
423
DEFINE_Bool(enable_low_cardinality_optimize, "true");
424
DEFINE_Bool(enable_low_cardinality_cache_code, "true");
425
426
// be policy
427
// whether check compaction checksum
428
DEFINE_mBool(enable_compaction_checksum, "false");
429
// whether disable automatic compaction task
430
DEFINE_mBool(disable_auto_compaction, "false");
431
// whether enable vertical compaction
432
DEFINE_mBool(enable_vertical_compaction, "true");
433
// whether enable ordered data compaction
434
DEFINE_mBool(enable_ordered_data_compaction, "true");
435
// In vertical compaction, column number for every group
436
DEFINE_mInt32(vertical_compaction_num_columns_per_group, "5");
437
// In vertical compaction, max memory usage for row_source_buffer
438
DEFINE_Int32(vertical_compaction_max_row_source_memory_mb, "1024");
439
// In vertical compaction, max dest segment file size
440
DEFINE_mInt64(vertical_compaction_max_segment_size, "1073741824");
441
442
// If enabled, segments will be flushed column by column
443
DEFINE_mBool(enable_vertical_segment_writer, "true");
444
445
// In ordered data compaction, min segment size for input rowset
446
DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760");
447
448
// This config can be set to limit thread number in compaction thread pool.
449
DEFINE_mInt32(max_base_compaction_threads, "4");
450
DEFINE_mInt32(max_cumu_compaction_threads, "-1");
451
DEFINE_mInt32(max_single_replica_compaction_threads, "-1");
452
453
DEFINE_Bool(enable_base_compaction_idle_sched, "true");
454
DEFINE_mInt64(base_compaction_min_rowset_num, "5");
455
DEFINE_mInt64(base_compaction_max_compaction_score, "20");
456
DEFINE_mInt64(mow_base_compaction_max_compaction_score, "200");
457
DEFINE_mDouble(base_compaction_min_data_ratio, "0.3");
458
DEFINE_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
459
460
DEFINE_Bool(enable_skip_tablet_compaction, "true");
461
DEFINE_mInt32(skip_tablet_compaction_second, "10");
462
463
// output rowset of cumulative compaction total disk size exceed this config size,
464
// this rowset will be given to base compaction, unit is m byte.
465
DEFINE_mInt64(compaction_promotion_size_mbytes, "1024");
466
467
// output rowset of cumulative compaction total disk size exceed this config ratio of
468
// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
469
// 0 and 1.
470
DEFINE_mDouble(compaction_promotion_ratio, "0.05");
471
472
// the smallest size of rowset promotion. When the rowset is less than this config, this
473
// rowset will be not given to base compaction. The unit is m byte.
474
DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128");
475
476
// When output rowset of cumulative compaction total version count (end_version - start_version)
477
// exceed this config count, the rowset will be moved to base compaction
478
// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
479
// related cost on delete bitmap more effectively.
480
DEFINE_mInt64(compaction_promotion_version_count, "1000");
481
482
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
483
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
484
DEFINE_mInt64(compaction_min_size_mbytes, "64");
485
486
// cumulative compaction policy: min and max delta file's number
487
DEFINE_mInt64(cumulative_compaction_min_deltas, "5");
488
DEFINE_mInt64(cumulative_compaction_max_deltas, "1000");
489
DEFINE_mInt32(cumulative_compaction_max_deltas_factor, "10");
490
491
// This config can be set to limit thread number in  multiget thread pool.
492
DEFINE_mInt32(multi_get_max_threads, "10");
493
494
// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
495
DEFINE_mInt64(total_permits_for_compaction_score, "1000000");
496
497
// sleep interval in ms after generated compaction tasks
498
DEFINE_mInt32(generate_compaction_tasks_interval_ms, "100");
499
500
// sleep interval in second after update replica infos
501
DEFINE_mInt32(update_replica_infos_interval_seconds, "60");
502
503
// Compaction task number per disk.
504
// Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each.
505
DEFINE_mInt32(compaction_task_num_per_disk, "4");
506
// compaction thread num for fast disk(typically .SSD), must be greater than 2.
507
DEFINE_mInt32(compaction_task_num_per_fast_disk, "8");
508
DEFINE_Validator(compaction_task_num_per_disk,
509
                 [](const int config) -> bool { return config >= 2; });
510
DEFINE_Validator(compaction_task_num_per_fast_disk,
511
                 [](const int config) -> bool { return config >= 2; });
512
DEFINE_Validator(low_priority_compaction_task_num_per_disk,
513
                 [](const int config) -> bool { return config >= 2; });
514
515
// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
516
DEFINE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9");
517
// Minimum number of threads required in the thread pool to activate the large cumu compaction delay strategy.
518
// The delay strategy is only applied when the thread pool has at least this many threads.
519
// Default -1 means disable.
520
DEFINE_mInt32(large_cumu_compaction_task_min_thread_num, "-1");
521
// Maximum size threshold (in bytes) for input rowsets. Compaction tasks with input size
522
// exceeding this threshold will be delayed when thread pool is near capacity. Default 512MB.
523
DEFINE_mInt32(large_cumu_compaction_task_bytes_threshold, "536870912");
524
// Maximum row count threshold for compaction input. Compaction tasks with row count
525
// exceeding this threshold will be delayed when thread pool is near capacity. Default 1 million.
526
DEFINE_mInt32(large_cumu_compaction_task_row_num_threshold, "1000000");
527
528
// Not compact the invisible versions, but with some limitations:
529
// if not timeout, keep no more than compaction_keep_invisible_version_max_count versions;
530
// if timeout, keep no more than compaction_keep_invisible_version_min_count versions.
531
DEFINE_mInt32(compaction_keep_invisible_version_timeout_sec, "1800");
532
DEFINE_mInt32(compaction_keep_invisible_version_min_count, "50");
533
DEFINE_mInt32(compaction_keep_invisible_version_max_count, "500");
534
535
// Threshold to logging compaction trace, in seconds.
536
DEFINE_mInt32(base_compaction_trace_threshold, "60");
537
DEFINE_mInt32(cumulative_compaction_trace_threshold, "10");
538
DEFINE_mBool(disable_compaction_trace_log, "true");
539
540
// Interval to picking rowset to compact, in seconds
541
DEFINE_mInt64(pick_rowset_to_compact_interval_sec, "86400");
542
543
// Compaction priority schedule
544
DEFINE_mBool(enable_compaction_priority_scheduling, "true");
545
DEFINE_mInt32(low_priority_compaction_task_num_per_disk, "2");
546
DEFINE_mInt32(low_priority_compaction_score_threshold, "200");
547
548
// Thread count to do tablet meta checkpoint, -1 means use the data directories count.
549
DEFINE_Int32(max_meta_checkpoint_threads, "-1");
550
551
// Threshold to logging agent task trace, in seconds.
552
DEFINE_mInt32(agent_task_trace_threshold_sec, "2");
553
554
// This config can be set to limit thread number in tablet migration thread pool.
555
DEFINE_Int32(min_tablet_migration_threads, "1");
556
DEFINE_Int32(max_tablet_migration_threads, "1");
557
558
DEFINE_mInt32(finished_migration_tasks_size, "10000");
559
// If size less than this, the remaining rowsets will be force to complete
560
DEFINE_mInt32(migration_remaining_size_threshold_mb, "10");
561
// If the task runs longer than this time, the task will be terminated, in seconds.
562
// timeout = std::max(migration_task_timeout_secs,  tablet size / 1MB/s)
563
DEFINE_mInt32(migration_task_timeout_secs, "300");
564
// timeout for try_lock migration lock
565
DEFINE_Int64(migration_lock_timeout_ms, "1000");
566
567
// Port to start debug webserver on
568
DEFINE_Int32(webserver_port, "8040");
569
// Https enable flag
570
DEFINE_Bool(enable_https, "false");
571
// Path of certificate
572
DEFINE_String(ssl_certificate_path, "");
573
// Path of private key
574
DEFINE_String(ssl_private_key_path, "");
575
// Whether to check authorization
576
DEFINE_Bool(enable_all_http_auth, "false");
577
// Number of webserver workers
578
DEFINE_Int32(webserver_num_workers, "128");
579
580
DEFINE_Bool(enable_single_replica_load, "true");
581
// Number of download workers for single replica load
582
DEFINE_Int32(single_replica_load_download_num_workers, "64");
583
584
// Used for mini Load. mini load data file will be removed after this time.
585
DEFINE_Int64(load_data_reserve_hours, "4");
586
// log error log will be removed after this time
587
DEFINE_mInt64(load_error_log_reserve_hours, "48");
588
// error log size limit, default 200MB
589
DEFINE_mInt64(load_error_log_limit_bytes, "209715200");
590
591
DEFINE_Int32(brpc_heavy_work_pool_threads, "-1");
592
DEFINE_Int32(brpc_light_work_pool_threads, "-1");
593
DEFINE_Int32(brpc_heavy_work_pool_max_queue_size, "-1");
594
DEFINE_Int32(brpc_light_work_pool_max_queue_size, "-1");
595
DEFINE_mBool(enable_bthread_transmit_block, "true");
596
DEFINE_Int32(brpc_arrow_flight_work_pool_threads, "-1");
597
DEFINE_Int32(brpc_arrow_flight_work_pool_max_queue_size, "-1");
598
599
//Enable brpc builtin services, see:
600
//https://brpc.apache.org/docs/server/basics/#disable-built-in-services-completely
601
DEFINE_Bool(enable_brpc_builtin_services, "true");
602
603
// Enable brpc connection check
604
DEFINE_Bool(enable_brpc_connection_check, "false");
605
606
DEFINE_mInt64(brpc_connection_check_timeout_ms, "10000");
607
608
// The maximum amount of data that can be processed by a stream load
609
DEFINE_mInt64(streaming_load_max_mb, "102400");
610
// Some data formats, such as JSON, cannot be streamed.
611
// Therefore, it is necessary to limit the maximum number of
612
// such data when using stream load to prevent excessive memory consumption.
613
DEFINE_mInt64(streaming_load_json_max_mb, "100");
614
// the alive time of a TabletsChannel.
615
// If the channel does not receive any data till this time,
616
// the channel will be removed.
617
DEFINE_mInt32(streaming_load_rpc_max_alive_time_sec, "1200");
618
// the timeout of a rpc to open the tablet writer in remote BE.
619
// short operation time, can set a short timeout
620
DEFINE_Int32(tablet_writer_open_rpc_timeout_sec, "60");
621
// You can ignore brpc error '[E1011]The server is overcrowded' when writing data.
622
DEFINE_mBool(tablet_writer_ignore_eovercrowded, "true");
623
DEFINE_mInt32(slave_replica_writer_rpc_timeout_sec, "60");
624
// Whether to enable stream load record function, the default is false.
625
// False: disable stream load record
626
DEFINE_mBool(enable_stream_load_record, "false");
627
// batch size of stream load record reported to FE
628
DEFINE_mInt32(stream_load_record_batch_size, "50");
629
// expire time of stream load record in rocksdb.
630
DEFINE_Int32(stream_load_record_expire_time_secs, "28800");
631
// time interval to clean expired stream load records
632
DEFINE_mInt64(clean_stream_load_record_interval_secs, "1800");
633
// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
634
DEFINE_mBool(enable_stream_load_commit_txn_on_be, "false");
635
// The buffer size to store stream table function schema info
636
DEFINE_Int64(stream_tvf_buffer_size, "1048576"); // 1MB
637
638
// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
639
// You may need to lower the speed when the sink receiver bes are too busy.
640
DEFINE_mInt32(olap_table_sink_send_interval_microseconds, "1000");
641
DEFINE_mDouble(olap_table_sink_send_interval_auto_partition_factor, "0.001");
642
643
// Fragment thread pool
644
DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_min, "16");
645
DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_max, "512");
646
DEFINE_Int32(fragment_mgr_async_work_pool_queue_size, "4096");
647
648
// Control the number of disks on the machine.  If 0, this comes from the system settings.
649
DEFINE_Int32(num_disks, "0");
650
// The read size is the size of the reads sent to os.
651
// There is a trade off of latency and throughout, trying to keep disks busy but
652
// not introduce seeks.  The literature seems to agree that with 8 MB reads, random
653
// io and sequential io perform similarly.
654
DEFINE_Int32(read_size, "8388608");    // 8 * 1024 * 1024, Read Size (in bytes)
655
DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes)
656
657
// for pprof
658
DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
659
// for jeprofile in jemalloc
660
DEFINE_mString(jeprofile_dir, "${DORIS_HOME}/log");
661
DEFINE_mBool(enable_je_purge_dirty_pages, "true");
662
DEFINE_mInt32(je_dirty_decay_ms, "5000");
663
664
// to forward compatibility, will be removed later
665
DEFINE_mBool(enable_token_check, "true");
666
667
// to open/close system metrics
668
DEFINE_Bool(enable_system_metrics, "true");
669
670
// Number of cores Doris will used, this will effect only when it's greater than 0.
671
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
672
DEFINE_Int32(num_cores, "0");
673
674
// When BE start, If there is a broken disk, BE process will exit by default.
675
// Otherwise, we will ignore the broken disk,
676
DEFINE_Bool(ignore_broken_disk, "false");
677
DEFINE_Bool(ignore_file_cache_dir_upgrade_failure, "false");
678
679
// Sleep time in milliseconds between memory maintenance iterations
680
DEFINE_mInt32(memory_maintenance_sleep_time_ms, "50");
681
682
// Memory gc are expensive, wait a while to avoid too frequent.
683
DEFINE_mInt32(memory_gc_sleep_time_ms, "500");
684
685
// max write buffer size before flush, default 200MB
686
DEFINE_mInt64(write_buffer_size, "209715200");
687
// max buffer size used in memtable for the aggregated table, default 400MB
688
DEFINE_mInt64(write_buffer_size_for_agg, "419430400");
689
// max parallel flush task per memtable writer
690
DEFINE_mInt32(memtable_flush_running_count_limit, "2");
691
692
// maximum sleep time to wait for memory when writing or flushing memtable.
693
DEFINE_mInt32(memtable_wait_for_memory_sleep_time_s, "300");
694
695
DEFINE_Int32(load_process_max_memory_limit_percent, "50"); // 50%
696
697
// If the memory consumption of load jobs exceed load_process_max_memory_limit,
698
// all load jobs will hang there to wait for memtable flush. We should have a
699
// soft limit which can trigger the memtable flush for the load channel who
700
// consumes lagest memory size before we reach the hard limit. The soft limit
701
// might avoid all load jobs hang at the same time.
702
DEFINE_Int32(load_process_soft_mem_limit_percent, "80");
703
704
// If load memory consumption is within load_process_safe_mem_permit_percent,
705
// memtable memory limiter will do nothing.
706
DEFINE_Int32(load_process_safe_mem_permit_percent, "5");
707
708
// If there are a lot of memtable memory, then wait them flush finished.
709
DEFINE_mDouble(load_max_wg_active_memtable_percent, "0.6");
710
711
// result buffer cancelled time (unit: second)
712
DEFINE_mInt32(result_buffer_cancelled_interval_time, "300");
713
714
// arrow flight result sink buffer rows size, default 4096 * 8
715
DEFINE_mInt32(arrow_flight_result_sink_buffer_size_rows, "32768");
716
// The timeout for ADBC Client to wait for data using arrow flight reader.
717
// If the query is very complex and no result is generated after this time, consider increasing this timeout.
718
DEFINE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms, "300000");
719
720
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
721
DEFINE_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
722
723
// sync tablet_meta when modifying meta
724
DEFINE_mBool(sync_tablet_meta, "false");
725
726
// sync when closing a file writer
727
DEFINE_mBool(sync_file_on_close, "true");
728
729
// default thrift rpc timeout ms
730
DEFINE_mInt32(thrift_rpc_timeout_ms, "60000");
731
732
// txn commit rpc timeout
733
DEFINE_mInt32(txn_commit_rpc_timeout_ms, "180000");
734
735
// If set to true, metric calculator will run
736
DEFINE_Bool(enable_metric_calculator, "true");
737
738
// max consumer num in one data consumer group, for routine load
739
DEFINE_mInt32(max_consumer_num_per_group, "3");
740
741
// the max size of thread pool for routine load task.
742
// this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5)
743
DEFINE_Int32(max_routine_load_thread_pool_size, "1024");
744
745
// the timeout of condition variable wait in blocking_get and blocking_put
746
DEFINE_mInt32(blocking_queue_cv_wait_timeout_ms, "1000");
747
748
// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
749
// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
750
DEFINE_mInt32(max_memory_sink_batch_count, "20");
751
752
// This configuration is used for the context gc thread schedule period
753
// note: unit is minute, default is 5min
754
DEFINE_mInt32(scan_context_gc_interval_min, "5");
755
756
// es scroll keep-alive
757
DEFINE_String(es_scroll_keepalive, "5m");
758
759
// HTTP connection timeout for es
760
DEFINE_mInt32(es_http_timeout_ms, "5000");
761
762
// the max client cache number per each host
763
// There are variety of client cache in BE, but currently we use the
764
// same cache size configuration.
765
// TODO(cmy): use different config to set different client cache if necessary.
766
DEFINE_Int32(max_client_cache_size_per_host, "10");
767
768
DEFINE_Int32(max_master_fe_client_cache_size, "10");
769
770
// Dir to save files downloaded by SmallFileMgr
771
DEFINE_String(small_file_dir, "${DORIS_HOME}/lib/small_file/");
772
// path gc
773
DEFINE_Bool(path_gc_check, "true");
774
DEFINE_mInt32(path_gc_check_interval_second, "86400");
775
DEFINE_mInt32(path_gc_check_step, "1000");
776
DEFINE_mInt32(path_gc_check_step_interval_ms, "10");
777
778
// The following 2 configs limit the max usage of disk capacity of a data dir.
779
// If both of these 2 threshold reached, no more data can be writen into that data dir.
780
// The percent of max used capacity of a data dir
781
DEFINE_mInt32(storage_flood_stage_usage_percent, "90"); // 90%
782
// The min bytes that should be left of a data dir
783
DEFINE_mInt64(storage_flood_stage_left_capacity_bytes, "1073741824"); // 1GB
784
// number of thread for flushing memtable per store
785
DEFINE_Int32(flush_thread_num_per_store, "6");
786
// number of thread for flushing memtable per store, for high priority load task
787
DEFINE_Int32(high_priority_flush_thread_num_per_store, "6");
788
// number of threads = min(flush_thread_num_per_store * num_store,
789
//                         max_flush_thread_num_per_cpu * num_cpu)
790
DEFINE_Int32(max_flush_thread_num_per_cpu, "4");
791
792
// config for tablet meta checkpoint
793
DEFINE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num, "10");
794
DEFINE_mInt32(tablet_meta_checkpoint_min_interval_secs, "600");
795
DEFINE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs, "600");
796
797
// config for default rowset type
798
// Valid configs: ALPHA, BETA
799
DEFINE_String(default_rowset_type, "BETA");
800
801
// Maximum size of a single message body in all protocols
802
DEFINE_Int64(brpc_max_body_size, "3147483648");
803
DEFINE_Int64(brpc_socket_max_unwritten_bytes, "-1");
804
// TODO(zxy): expect to be true in v1.3
805
// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
806
// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
807
// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
808
DEFINE_mBool(transfer_large_data_by_brpc, "true");
809
810
// max number of txns for every txn_partition_map in txn manager
811
// this is a self protection to avoid too many txns saving in manager
812
DEFINE_mInt64(max_runnings_transactions_per_txn_map, "2000");
813
814
// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
815
// this is a an enhancement for better performance to manage tablet
816
DEFINE_Int32(tablet_map_shard_size, "256");
817
818
// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
819
// this is a an enhancement for better performance to manage txn
820
DEFINE_Int32(txn_map_shard_size, "1024");
821
822
// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
823
// this is a an enhancement for better performance to commit and publish txn
824
DEFINE_Int32(txn_shard_size, "1024");
825
826
// Whether to continue to start be when load tablet from header failed.
827
DEFINE_Bool(ignore_load_tablet_failure, "false");
828
829
// Whether to continue to start be when load tablet from header failed.
830
DEFINE_mBool(ignore_rowset_stale_unconsistent_delete, "false");
831
832
// Set max cache's size of query results, the unit is M byte
833
DEFINE_Int32(query_cache_max_size_mb, "256");
834
835
// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
836
DEFINE_Int32(query_cache_elasticity_size_mb, "128");
837
838
// Maximum number of cache partitions corresponding to a SQL
839
DEFINE_Int32(query_cache_max_partition_count, "1024");
840
841
// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
842
// the load process will reject new incoming load job of this tablet.
843
// This is to avoid too many version num.
844
DEFINE_mInt32(max_tablet_version_num, "2000");
845
846
DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
847
848
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
849
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
850
DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");
851
852
// disable zone map index when page row is too few
853
DEFINE_mInt32(zone_map_row_num_threshold, "20");
854
855
// aws sdk log level
856
//    Off = 0,
857
//    Fatal = 1,
858
//    Error = 2,
859
//    Warn = 3,
860
//    Info = 4,
861
//    Debug = 5,
862
//    Trace = 6
863
DEFINE_Int32(aws_log_level, "3");
864
DEFINE_Validator(aws_log_level,
865
                 [](const int config) -> bool { return config >= 0 && config <= 6; });
866
867
// azure sdk log level
868
//    Verbose = 1,
869
//    Informational = 2,
870
//    Warning = 3,
871
//    Error = 4
872
DEFINE_Int32(azure_log_level, "3");
873
DEFINE_Validator(azure_log_level,
874
                 [](const int config) -> bool { return config >= 1 && config <= 4; });
875
876
// the buffer size when read data from remote storage like s3
877
DEFINE_mInt32(remote_storage_read_buffer_mb, "16");
878
879
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
880
// smaller than this value will continue to accumulate. specified as number of bytes.
881
// Decreasing this value will increase the frequency of consume/release.
882
// Increasing this value will cause MemTracker statistics to be inaccurate.
883
DEFINE_mInt32(mem_tracker_consume_min_size_bytes, "1048576");
884
885
// The version information of the tablet will be stored in the memory
886
// in an adjacency graph data structure.
887
// And as the new version is written and the old version is deleted,
888
// the data structure will begin to have empty vertex with no edge associations(orphan vertex).
889
// This config is used to control that when the proportion of orphan vertex is greater than the threshold,
890
// the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely.
891
// This config usually only needs to be modified during testing.
892
// In most cases, it does not need to be modified.
893
DEFINE_mDouble(tablet_version_graph_orphan_vertex_ratio, "0.1");
894
895
// share delta writers when memtable_on_sink_node = true
896
DEFINE_Bool(share_delta_writers, "true");
897
// timeout for open load stream rpc in ms
898
DEFINE_Int64(open_load_stream_timeout_ms, "60000"); // 60s
899
// enable write background when using brpc stream
900
DEFINE_mBool(enable_brpc_stream_write_background, "true");
901
902
// brpc streaming max_buf_size in bytes
903
DEFINE_Int64(load_stream_max_buf_size, "20971520"); // 20MB
904
// brpc streaming messages_in_batch
905
DEFINE_Int32(load_stream_messages_in_batch, "128");
906
// brpc streaming StreamWait seconds on EAGAIN
907
DEFINE_Int32(load_stream_eagain_wait_seconds, "600");
908
// max tasks per flush token in load stream
909
DEFINE_Int32(load_stream_flush_token_max_tasks, "15");
910
// max wait flush token time in load stream
911
DEFINE_Int32(load_stream_max_wait_flush_token_time_ms, "600000");
912
// number of send batch thread pool size
913
DEFINE_Int32(send_batch_thread_pool_thread_num, "64");
914
// number of send batch thread pool queue size
915
DEFINE_Int32(send_batch_thread_pool_queue_size, "102400");
916
917
// Limit the number of segment of a newly created rowset.
918
// The newly created rowset may to be compacted after loading,
919
// so if there are too many segment in a rowset, the compaction process
920
// will run out of memory.
921
// When doing compaction, each segment may take at least 1MB buffer.
922
DEFINE_mInt32(max_segment_num_per_rowset, "1000");
923
DEFINE_mInt32(segment_compression_threshold_kb, "256");
924
925
// Time to clean up useless JDBC connection pool cache
926
DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800");
927
928
// Global bitmap cache capacity for aggregation cache, size in bytes
929
DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
930
// The default delete bitmap cache is set to 100MB,
931
// which can be insufficient and cause performance issues when the amount of user data is large.
932
// To mitigate the problem of an inadequate cache,
933
// we will take the larger of 1.0% of the total memory and 100MB as the delete bitmap cache size.
934
DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "1.0%");
935
DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
936
937
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
938
// If the dependent kafka broker version older than 0.10.0.0,
939
// the value of kafka_api_version_request should be false, and the
940
// value set by the fallback version kafka_broker_version_fallback will be used,
941
// and the valid values are: 0.9.0.x, 0.8.x.y.
942
DEFINE_String(kafka_api_version_request, "true");
943
DEFINE_String(kafka_broker_version_fallback, "0.10.0");
944
DEFINE_String(kafka_debug, "disable");
945
946
// The number of pool siz of routine load consumer.
947
// If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608
948
// Change this size to 0 to fix it temporarily.
949
DEFINE_mInt32(routine_load_consumer_pool_size, "1024");
950
951
// Used in single-stream-multi-table load. When receive a batch of messages from kafka,
952
// if the size of batch is more than this threshold, we will request plans for all related tables.
953
DEFINE_Int32(multi_table_batch_plan_threshold, "200");
954
955
// Used in single-stream-multi-table load. When receiving a batch of messages from Kafka,
956
// if the size of the table wait for plan is more than this threshold, we will request plans for all related tables.
957
// The param is aimed to avoid requesting and executing too many plans at once.
958
// Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC
959
// and improve the real-time processing of data.
960
DEFINE_Int32(multi_table_max_wait_tables, "5");
961
962
// When the timeout of a load task is less than this threshold,
963
// Doris treats it as a high priority task.
964
// high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic.
965
// this threshold is mainly used to identify routine load tasks and should not be modified if not necessary.
966
DEFINE_mInt32(load_task_high_priority_threshold_second, "600");
967
968
// The min timeout of load rpc (add batch, close, etc.)
969
// Because a load rpc may be blocked for a while.
970
// Increase this config may avoid rpc timeout.
971
DEFINE_mInt32(min_load_rpc_timeout_ms, "20000");
972
973
// use which protocol to access function service, candicate is baidu_std/h2:grpc
974
DEFINE_String(function_service_protocol, "h2:grpc");
975
976
// use which load balancer to select server to connect
977
DEFINE_String(rpc_load_balancer, "rr");
978
979
// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
980
// so we set a soft limit, default is 10MB
981
DEFINE_Int32(string_type_length_soft_limit_bytes, "10485760");
982
983
DEFINE_Validator(string_type_length_soft_limit_bytes,
984
                 [](const int config) -> bool { return config > 0 && config <= 2147483643; });
985
986
// Threshold of reading a small file into memory
987
DEFINE_mInt32(in_memory_file_size, "1048576"); // 1MB
988
989
// Max size of parquet page header in bytes
990
DEFINE_mInt32(parquet_header_max_size_mb, "1");
991
// Max buffer size for parquet row group
992
DEFINE_mInt32(parquet_rowgroup_max_buffer_mb, "128");
993
// Max buffer size for parquet chunk column
994
DEFINE_mInt32(parquet_column_max_buffer_mb, "8");
995
DEFINE_mDouble(max_amplified_read_ratio, "0.8");
996
DEFINE_mInt32(merged_oss_min_io_size, "1048576");
997
DEFINE_mInt32(merged_hdfs_min_io_size, "8192");
998
999
// OrcReader
1000
DEFINE_mInt32(orc_natural_read_size_mb, "8");
1001
DEFINE_mInt64(big_column_size_buffer, "65535");
1002
DEFINE_mInt64(small_column_size_buffer, "100");
1003
1004
// Perform the always_true check at intervals determined by runtime_filter_sampling_frequency
1005
DEFINE_mInt32(runtime_filter_sampling_frequency, "64");
1006
DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600");
1007
DEFINE_mBool(execution_ignore_eovercrowded, "true");
1008
// cooldown task configs
1009
DEFINE_Int32(cooldown_thread_num, "5");
1010
DEFINE_mInt64(generate_cooldown_task_interval_sec, "20");
1011
DEFINE_mInt32(remove_unused_remote_files_interval_sec, "21600"); // 6h
1012
DEFINE_mInt32(confirm_unused_remote_files_interval_sec, "60");
1013
DEFINE_Int32(cold_data_compaction_thread_num, "2");
1014
DEFINE_mInt32(cold_data_compaction_interval_sec, "1800");
1015
1016
DEFINE_String(tmp_file_dir, "tmp");
1017
1018
DEFINE_Int32(min_s3_file_system_thread_num, "16");
1019
DEFINE_Int32(max_s3_file_system_thread_num, "64");
1020
1021
DEFINE_Bool(enable_time_lut, "true");
1022
DEFINE_mBool(enable_simdjson_reader, "true");
1023
1024
DEFINE_mBool(enable_query_like_bloom_filter, "true");
1025
// number of s3 scanner thread pool size
1026
DEFINE_Int32(doris_remote_scanner_thread_pool_thread_num, "48");
1027
// number of s3 scanner thread pool queue size
1028
DEFINE_Int32(doris_remote_scanner_thread_pool_queue_size, "102400");
1029
DEFINE_mInt64(block_cache_wait_timeout_ms, "1000");
1030
1031
// limit the queue of pending batches which will be sent by a single nodechannel
1032
DEFINE_mInt64(nodechannel_pending_queue_max_bytes, "67108864");
1033
1034
// The batch size for sending data by brpc streaming client
1035
DEFINE_mInt64(brpc_streaming_client_batch_bytes, "262144");
1036
1037
// Max waiting time to wait the "plan fragment start" rpc.
1038
// If timeout, the fragment will be cancelled.
1039
// This parameter is usually only used when the FE loses connection,
1040
// and the BE can automatically cancel the relevant fragment after the timeout,
1041
// so as to avoid occupying the execution thread for a long time.
1042
DEFINE_mInt32(max_fragment_start_wait_time_seconds, "30");
1043
1044
DEFINE_mInt32(fragment_mgr_cancel_worker_interval_seconds, "1");
1045
1046
// Node role tag for backend. Mix role is the default role, and computation role have no
1047
// any tablet.
1048
DEFINE_String(be_node_role, "mix");
1049
1050
// Hide webserver page for safety.
1051
// Hide the be config page for webserver.
1052
DEFINE_Bool(hide_webserver_config_page, "false");
1053
1054
DEFINE_Bool(enable_segcompaction, "true");
1055
1056
// Max number of segments allowed in a single segcompaction task.
1057
DEFINE_mInt32(segcompaction_batch_size, "10");
1058
1059
// Max row count allowed in a single source segment, bigger segments will be skipped.
1060
DEFINE_Int32(segcompaction_candidate_max_rows, "1048576");
1061
1062
// Max file size allowed in a single source segment, bigger segments will be skipped.
1063
DEFINE_Int64(segcompaction_candidate_max_bytes, "104857600");
1064
1065
// Max total row count allowed in a single segcompaction task.
1066
DEFINE_Int32(segcompaction_task_max_rows, "1572864");
1067
1068
// Max total file size allowed in a single segcompaction task.
1069
DEFINE_Int64(segcompaction_task_max_bytes, "157286400");
1070
1071
// Global segcompaction thread pool size.
1072
DEFINE_mInt32(segcompaction_num_threads, "5");
1073
1074
DEFINE_mInt32(segcompaction_wait_for_dbm_task_timeout_s, "3600"); // 1h
1075
1076
// enable java udf and jdbc scannode
1077
DEFINE_Bool(enable_java_support, "true");
1078
1079
// Set config randomly to check more issues in github workflow
1080
DEFINE_Bool(enable_fuzzy_mode, "false");
1081
1082
DEFINE_Bool(enable_debug_points, "false");
1083
1084
DEFINE_Int32(pipeline_executor_size, "0");
1085
DEFINE_Bool(enable_workload_group_for_scan, "false");
1086
DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");
1087
1088
// Whether use schema dict in backend side instead of MetaService side(cloud mode)
1089
DEFINE_mBool(variant_use_cloud_schema_dict_cache, "true");
1090
DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1");
1091
DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048");
1092
DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false");
1093
1094
// block file cache
1095
DEFINE_Bool(enable_file_cache, "false");
1096
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
1097
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
1098
// format: {"path": "/path/to/file_cache", "total_size":53687091200, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}
1099
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
1100
// Note1: storage is "disk" by default
1101
// Note2: when the storage is "memory", the path is ignored. So you can set xxx to anything you like
1102
// and doris will just reset the path to "memory" internally.
1103
// In a very wierd case when your storage is disk, and the directory, by accident, is named
1104
// "memory" for some reason, you should write the path as:
1105
//     {"path": "memory", "total_size":53687091200, "storage": "disk"}
1106
// or use the default storage value:
1107
//     {"path": "memory", "total_size":53687091200}
1108
// Both will use the directory "memory" on the disk instead of the real RAM.
1109
DEFINE_String(file_cache_path, "[{\"path\":\"${DORIS_HOME}/file_cache\"}]");
1110
DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB
1111
1112
DEFINE_Bool(clear_file_cache, "false");
1113
DEFINE_Bool(enable_file_cache_query_limit, "false");
1114
DEFINE_mInt32(file_cache_enter_disk_resource_limit_mode_percent, "90");
1115
DEFINE_mInt32(file_cache_exit_disk_resource_limit_mode_percent, "88");
1116
DEFINE_mBool(enable_evict_file_cache_in_advance, "true");
1117
DEFINE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent, "88");
1118
DEFINE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent, "85");
1119
DEFINE_mInt32(file_cache_evict_in_advance_interval_ms, "1000");
1120
DEFINE_mInt64(file_cache_evict_in_advance_batch_bytes, "31457280"); // 30MB
1121
DEFINE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold, "1000");
1122
1123
DEFINE_mBool(enable_read_cache_file_directly, "false");
1124
DEFINE_mBool(file_cache_enable_evict_from_other_queue_by_size, "true");
1125
// If true, evict the ttl cache using LRU when full.
1126
// Otherwise, only expiration can evict ttl and new data won't add to cache when full.
1127
DEFINE_Bool(enable_ttl_cache_evict_using_lru, "true");
1128
DEFINE_mBool(enbale_dump_error_file, "false");
1129
// limit the max size of error log on disk
1130
DEFINE_mInt64(file_cache_error_log_limit_bytes, "209715200"); // 200MB
1131
DEFINE_mInt64(cache_lock_wait_long_tail_threshold_us, "30000000");
1132
DEFINE_mInt64(cache_lock_held_long_tail_threshold_us, "30000000");
1133
DEFINE_mBool(enable_file_cache_keep_base_compaction_output, "false");
1134
DEFINE_mInt64(file_cache_remove_block_qps_limit, "1000");
1135
DEFINE_mInt64(file_cache_background_gc_interval_ms, "100");
1136
DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true");
1137
DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000");
1138
DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "3000");
1139
DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000");
1140
DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000");
1141
// dump queue only if the queue update specific times through several dump intervals
1142
DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000");
1143
DEFINE_mInt64(file_cache_background_lru_dump_tail_record_num, "5000000");
1144
DEFINE_mInt64(file_cache_background_lru_log_replay_interval_ms, "1000");
1145
DEFINE_mBool(enable_evaluate_shadow_queue_diff, "false");
1146
1147
DEFINE_Int32(file_cache_downloader_thread_num_min, "32");
1148
DEFINE_Int32(file_cache_downloader_thread_num_max, "32");
1149
1150
DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800");
1151
DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600");
1152
DEFINE_mBool(enable_write_index_searcher_cache, "false");
1153
// inverted index searcher cache size
1154
DEFINE_String(inverted_index_searcher_cache_limit, "10%");
1155
DEFINE_Bool(enable_inverted_index_cache_check_timestamp, "true");
1156
DEFINE_mBool(enable_inverted_index_correct_term_write, "true");
1157
DEFINE_Int32(inverted_index_fd_number_limit_percent, "20"); // 20%
1158
DEFINE_Int32(inverted_index_query_cache_shards, "256");
1159
1160
// inverted index match bitmap cache size
1161
DEFINE_String(inverted_index_query_cache_limit, "10%");
1162
1163
// inverted index
1164
DEFINE_mDouble(inverted_index_ram_buffer_size, "512");
1165
// -1 indicates not working.
1166
// Normally we should not change this, it's useful for testing.
1167
DEFINE_mInt32(inverted_index_max_buffered_docs, "-1");
1168
// dict path for chinese analyzer
1169
DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict");
1170
DEFINE_Int32(inverted_index_read_buffer_size, "4096");
1171
// tree depth for bkd index
1172
DEFINE_Int32(max_depth_in_bkd_tree, "32");
1173
// index compaction
1174
DEFINE_mBool(inverted_index_compaction_enable, "true");
1175
// Only for debug, do not use in production
1176
DEFINE_mBool(debug_inverted_index_compaction, "false");
1177
// index by RAM directory
1178
DEFINE_mBool(inverted_index_ram_dir_enable, "true");
1179
// wheather index by RAM directory when base compaction
1180
DEFINE_mBool(inverted_index_ram_dir_enable_when_base_compaction, "true");
1181
// use num_broadcast_buffer blocks as buffer to do broadcast
1182
DEFINE_Int32(num_broadcast_buffer, "32");
1183
1184
// max depth of expression tree allowed.
1185
DEFINE_Int32(max_depth_of_expr_tree, "600");
1186
1187
// Report a tablet as bad when io errors occurs more than this value.
1188
DEFINE_mInt64(max_tablet_io_errors, "-1");
1189
1190
// Report a tablet as bad when its path not found
1191
DEFINE_Int32(tablet_path_check_interval_seconds, "-1");
1192
DEFINE_mInt32(tablet_path_check_batch_size, "1000");
1193
1194
// it must be larger than or equal to 5MB
1195
DEFINE_mInt64(s3_write_buffer_size, "5242880");
1196
// Log interval when doing s3 upload task
1197
DEFINE_mInt32(s3_file_writer_log_interval_second, "60");
1198
DEFINE_mInt64(file_cache_max_file_reader_cache_size, "1000000");
1199
DEFINE_mInt64(hdfs_write_batch_buffer_size_mb, "1"); // 1MB
1200
1201
//disable shrink memory by default
1202
DEFINE_mBool(enable_shrink_memory, "false");
1203
DEFINE_mInt32(schema_cache_capacity, "1024");
1204
DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
1205
1206
// max number of segment cache, default -1 for backward compatibility fd_number*2/5
1207
DEFINE_Int32(segment_cache_capacity, "-1");
1208
DEFINE_Int32(segment_cache_fd_percentage, "20");
1209
DEFINE_mInt32(estimated_mem_per_column_reader, "512");
1210
DEFINE_Int32(segment_cache_memory_percentage, "5");
1211
DEFINE_Bool(enable_segment_cache_prune, "true");
1212
1213
// enable feature binlog, default false
1214
DEFINE_Bool(enable_feature_binlog, "false");
1215
1216
// enable set in BitmapValue
1217
DEFINE_Bool(enable_set_in_bitmap_value, "true");
1218
1219
DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000");
1220
DEFINE_Int32(max_hdfs_file_handle_cache_time_sec, "28800");
1221
DEFINE_Int64(max_external_file_meta_cache_num, "1000");
1222
DEFINE_mInt32(common_obj_lru_cache_stale_sweep_time_sec, "900");
1223
// Apply delete pred in cumu compaction
1224
DEFINE_mBool(enable_delete_when_cumu_compaction, "false");
1225
1226
// max_write_buffer_number for rocksdb
1227
DEFINE_Int32(rocksdb_max_write_buffer_number, "5");
1228
1229
DEFINE_mBool(allow_zero_date, "false");
1230
DEFINE_Bool(allow_invalid_decimalv2_literal, "false");
1231
DEFINE_mString(kerberos_ccache_path, "/tmp/");
1232
DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf");
1233
// Deprecated
1234
DEFINE_mInt32(kerberos_refresh_interval_second, "43200");
1235
1236
// JDK-8153057: avoid StackOverflowError thrown from the UncaughtExceptionHandler in thread "process reaper"
1237
DEFINE_mBool(jdk_process_reaper_use_default_stack_size, "true");
1238
1239
DEFINE_mString(get_stack_trace_tool, "libunwind");
1240
DEFINE_mString(dwarf_location_info_mode, "FAST");
1241
DEFINE_mBool(enable_address_sanitizers_with_stack_trace, "true");
1242
1243
// the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
1244
DEFINE_mInt64(auto_inc_prefetch_size_ratio, "10");
1245
1246
// the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer
1247
DEFINE_mInt64(auto_inc_low_water_level_mark_size_ratio, "3");
1248
1249
// number of threads that fetch auto-inc ranges from FE
1250
DEFINE_mInt64(auto_inc_fetch_thread_num, "3");
1251
// default max to 2048 connections
1252
DEFINE_mInt64(lookup_connection_cache_capacity, "2048");
1253
1254
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
1255
DEFINE_mInt64(LZ4_HC_compression_level, "9");
1256
1257
DEFINE_mBool(enable_merge_on_write_correctness_check, "true");
1258
// USED FOR DEBUGING
1259
// core directly if the compaction found there's duplicate key on mow table
1260
DEFINE_mBool(enable_mow_compaction_correctness_check_core, "false");
1261
// USED FOR DEBUGING
1262
// let compaction fail if the compaction found there's duplicate key on mow table
1263
DEFINE_mBool(enable_mow_compaction_correctness_check_fail, "false");
1264
// rowid conversion correctness check when compaction for mow table
1265
DEFINE_mBool(enable_rowid_conversion_correctness_check, "false");
1266
// missing rows correctness check when compaction for mow table
1267
DEFINE_mBool(enable_missing_rows_correctness_check, "false");
1268
// When the number of missing versions is more than this value, do not directly
1269
// retry the publish and handle it through async publish.
1270
DEFINE_mInt32(mow_publish_max_discontinuous_version_num, "20");
1271
// When the size of primary keys in memory exceeds this value, finish current segment
1272
// and create a new segment, used in compaction. Default 50MB.
1273
DEFINE_mInt64(mow_primary_key_index_max_size_in_memory, "52428800");
1274
// When the version is not continuous for MOW table in publish phase and the gap between
1275
// current txn's publishing version and the max version of the tablet exceeds this value,
1276
// don't print warning log
1277
DEFINE_mInt32(publish_version_gap_logging_threshold, "200");
1278
// get agg by cache for mow table
1279
DEFINE_mBool(enable_mow_get_agg_by_cache, "true");
1280
// get agg correctness check for mow table
1281
DEFINE_mBool(enable_mow_get_agg_correctness_check_core, "false");
1282
DEFINE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap, "true");
1283
DEFINE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap, "false");
1284
1285
// The secure path with user files, used in the `local` table function.
1286
DEFINE_mString(user_files_secure_path, "${DORIS_HOME}");
1287
1288
DEFINE_Int32(fe_expire_duration_seconds, "60");
1289
1290
DEFINE_Int32(grace_shutdown_wait_seconds, "120");
1291
1292
DEFINE_Int16(bitmap_serialize_version, "1");
1293
1294
// group commit config
1295
DEFINE_String(group_commit_wal_path, "");
1296
DEFINE_Int32(group_commit_replay_wal_retry_num, "10");
1297
DEFINE_Int32(group_commit_replay_wal_retry_interval_seconds, "5");
1298
DEFINE_Int32(group_commit_replay_wal_retry_interval_max_seconds, "1800");
1299
DEFINE_Int32(group_commit_relay_wal_threads, "10");
1300
// This config can be set to limit thread number in group commit request fragment thread pool.
1301
DEFINE_Int32(group_commit_insert_threads, "10");
1302
DEFINE_Int32(group_commit_memory_rows_for_max_filter_ratio, "10000");
1303
DEFINE_Bool(wait_internal_group_commit_finish, "false");
1304
// Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
1305
DEFINE_mInt32(group_commit_queue_mem_limit, "67108864");
1306
// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
1307
// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
1308
DEFINE_String(group_commit_wal_max_disk_limit, "10%");
1309
DEFINE_Bool(group_commit_wait_replay_wal_finish, "false");
1310
1311
DEFINE_mInt32(scan_thread_nice_value, "0");
1312
DEFINE_mInt32(tablet_schema_cache_recycle_interval, "3600");
1313
DEFINE_mInt32(tablet_schema_cache_capacity, "102400");
1314
1315
DEFINE_Bool(exit_on_exception, "false");
1316
// This config controls whether the s3 file writer would flush cache asynchronously
1317
DEFINE_Bool(enable_flush_file_cache_async, "true");
1318
1319
// cgroup
1320
DEFINE_String(doris_cgroup_cpu_path, "");
1321
1322
DEFINE_mBool(enable_be_proc_monitor, "false");
1323
DEFINE_mInt32(be_proc_monitor_interval_ms, "10000");
1324
1325
DEFINE_Int32(workload_group_metrics_interval_ms, "5000");
1326
1327
DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
1328
1329
// Ingest binlog work pool size, -1 is disable, 0 is hardware concurrency
1330
DEFINE_Int32(ingest_binlog_work_pool_size, "-1");
1331
1332
// Ingest binlog with persistent connection
1333
DEFINE_Bool(enable_ingest_binlog_with_persistent_connection, "false");
1334
1335
// Log ingest binlog elapsed threshold, -1 is disabled
1336
DEFINE_mInt64(ingest_binlog_elapsed_threshold_ms, "-1");
1337
1338
// Download binlog rate limit, unit is KB/s, 0 means no limit
1339
DEFINE_Int32(download_binlog_rate_limit_kbs, "0");
1340
1341
DEFINE_mInt32(buffered_reader_read_timeout_ms, "600000");
1342
1343
DEFINE_Bool(enable_snapshot_action, "false");
1344
1345
DEFINE_mInt32(variant_max_merged_tablet_schema_size, "2048");
1346
1347
DEFINE_mBool(enable_column_type_check, "true");
1348
// 128 MB
1349
DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
1350
1351
// Default 300s, if its value <= 0, then log is disabled
1352
DEFINE_mInt64(enable_debug_log_timeout_secs, "0");
1353
1354
// Tolerance for the number of partition id 0 in rowset, default 0
1355
DEFINE_Int32(ignore_invalid_partition_id_rowset_num, "0");
1356
1357
DEFINE_mInt32(report_query_statistics_interval_ms, "3000");
1358
// 30s
1359
DEFINE_mInt32(query_statistics_reserve_timeout_ms, "30000");
1360
1361
DEFINE_mInt32(report_exec_status_thread_num, "5");
1362
1363
// consider two high usage disk at the same available level if they do not exceed this diff.
1364
DEFINE_mDouble(high_disk_avail_level_diff_usages, "0.15");
1365
1366
// create tablet in partition random robin idx lru size, default 10000
1367
DEFINE_Int32(partition_disk_index_lru_size, "10000");
1368
// limit the storage space that query spill files can use
1369
DEFINE_String(spill_storage_root_path, "");
1370
DEFINE_String(spill_storage_limit, "20%");    // 20%
1371
DEFINE_mInt32(spill_gc_interval_ms, "2000");  // 2s
1372
DEFINE_mInt32(spill_gc_work_time_ms, "2000"); // 2s
1373
DEFINE_Int32(spill_io_thread_pool_thread_num, "-1");
1374
DEFINE_Validator(spill_io_thread_pool_thread_num, [](const int config) -> bool {
1375
    if (config == -1) {
1376
        CpuInfo::init();
1377
        spill_io_thread_pool_thread_num = std::max(48, CpuInfo::num_cores() * 2);
1378
    }
1379
    return true;
1380
});
1381
DEFINE_Int32(spill_io_thread_pool_queue_size, "102400");
1382
1383
// paused query in queue timeout(ms) will be resumed or canceled
1384
DEFINE_Int64(spill_in_paused_queue_timeout_ms, "60000");
1385
1386
DEFINE_mBool(check_segment_when_build_rowset_meta, "false");
1387
1388
DEFINE_mBool(force_azure_blob_global_endpoint, "false");
1389
1390
DEFINE_mInt32(max_s3_client_retry, "10");
1391
DEFINE_mInt32(s3_read_base_wait_time_ms, "100");
1392
DEFINE_mInt32(s3_read_max_wait_time_ms, "800");
1393
DEFINE_mBool(enable_s3_object_check_after_upload, "true");
1394
1395
DEFINE_mBool(enable_s3_rate_limiter, "false");
1396
DEFINE_mInt64(s3_get_bucket_tokens, "1000000000000000000");
1397
DEFINE_Validator(s3_get_bucket_tokens, [](int64_t config) -> bool { return config > 0; });
1398
1399
DEFINE_mInt64(s3_get_token_per_second, "1000000000000000000");
1400
DEFINE_Validator(s3_get_token_per_second, [](int64_t config) -> bool { return config > 0; });
1401
1402
DEFINE_mInt64(s3_get_token_limit, "0");
1403
1404
DEFINE_mInt64(s3_put_bucket_tokens, "1000000000000000000");
1405
DEFINE_Validator(s3_put_bucket_tokens, [](int64_t config) -> bool { return config > 0; });
1406
1407
DEFINE_mInt64(s3_put_token_per_second, "1000000000000000000");
1408
DEFINE_Validator(s3_put_token_per_second, [](int64_t config) -> bool { return config > 0; });
1409
1410
DEFINE_mInt64(s3_put_token_limit, "0");
1411
1412
DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors");
1413
1414
// ca_cert_file is in this path by default, Normally no modification is required
1415
// ca cert default path is different from different OS
1416
DEFINE_mString(ca_cert_file_paths,
1417
               "/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;"
1418
               "/etc/ssl/ca-bundle.pem");
1419
1420
/** Table sink configurations(currently contains only external table types) **/
1421
// Minimum data processed to scale writers in exchange when non partition writing
1422
DEFINE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold,
1423
              "26214400"); // 25MB
1424
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1425
DEFINE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold,
1426
              "26214400"); // 25MB
1427
// Minimum partition data processed to rebalance writers in exchange when partition writing
1428
DEFINE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold,
1429
              "15728640"); // 15MB
1430
// Maximum processed partition nums of per writer when partition writing
1431
DEFINE_mInt32(table_sink_partition_write_max_partition_nums_per_writer, "128");
1432
1433
/** Hive sink configurations **/
1434
DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB
1435
1436
/** Iceberg sink configurations **/
1437
DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB
1438
1439
DEFINE_mInt32(thrift_client_open_num_tries, "1");
1440
1441
DEFINE_Bool(enable_index_compaction, "false");
1442
1443
// http scheme in S3Client to use. E.g. http or https
1444
DEFINE_String(s3_client_http_scheme, "http");
1445
DEFINE_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
1446
    return config == "http" || config == "https";
1447
});
1448
1449
DEFINE_mBool(ignore_schema_change_check, "false");
1450
1451
DEFINE_mInt64(string_overflow_size, "4294967295"); // std::numic_limits<uint32_t>::max()
1452
1453
// The min thread num for BufferedReaderPrefetchThreadPool
1454
DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread, "16");
1455
// The max thread num for BufferedReaderPrefetchThreadPool
1456
DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread, "64");
1457
// The min thread num for S3FileUploadThreadPool
1458
DEFINE_Int64(num_s3_file_upload_thread_pool_min_thread, "16");
1459
// The max thread num for S3FileUploadThreadPool
1460
DEFINE_Int64(num_s3_file_upload_thread_pool_max_thread, "64");
1461
// The maximum jvm heap usage ratio for hdfs write workload
1462
DEFINE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio, "0.5");
1463
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
1464
DEFINE_mInt64(hdfs_jni_write_sleep_milliseconds, "300");
1465
// The max retry times when hdfs write failed
1466
DEFINE_mInt64(hdfs_jni_write_max_retry_time, "3");
1467
1468
// The min thread num for NonBlockCloseThreadPool
1469
DEFINE_Int64(min_nonblock_close_thread_num, "12");
1470
// The max thread num for NonBlockCloseThreadPool
1471
DEFINE_Int64(max_nonblock_close_thread_num, "64");
1472
// The possibility that mem allocator throws an exception during memory allocation
1473
// This config is for test usage, be careful when changing it.
1474
DEFINE_mDouble(mem_alloc_fault_probability, "0.0");
1475
// The time out milliseconds for remote fetch schema RPC, default 60s
1476
DEFINE_mInt64(fetch_remote_schema_rpc_timeout_ms, "60000");
1477
1478
DEFINE_Int64(s3_file_system_local_upload_buffer_size, "5242880");
1479
1480
//JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues. The default setting is off.
1481
DEFINE_Bool(enable_jvm_monitor, "false");
1482
1483
DEFINE_Int32(load_data_dirs_threads, "-1");
1484
1485
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
1486
DEFINE_mBool(skip_loading_stale_rowset_meta, "false");
1487
1488
DEFINE_Bool(enable_file_logger, "true");
1489
1490
// The minimum row group size when exporting Parquet files. default 128MB
1491
DEFINE_Int64(min_row_group_size, "134217728");
1492
1493
DEFINE_mInt64(compaction_memory_bytes_limit, "1073741824");
1494
1495
DEFINE_mInt64(compaction_batch_size, "-1");
1496
1497
// If set to false, the parquet reader will not use page index to filter data.
1498
// This is only for debug purpose, in case sometimes the page index
1499
// filter wrong data.
1500
DEFINE_mBool(enable_parquet_page_index, "true");
1501
1502
DEFINE_mBool(ignore_not_found_file_in_external_table, "true");
1503
1504
DEFINE_mBool(enable_hdfs_mem_limiter, "true");
1505
1506
DEFINE_mInt16(topn_agg_limit_multiplier, "2");
1507
1508
// Tablet meta size limit after serialization, 1.5GB
1509
DEFINE_mInt64(tablet_meta_serialize_size_limit, "1610612736");
1510
// Protobuf supports a maximum of 2GB, so the size of the tablet meta after serialization must be less than 2GB
1511
// 1717986918 = 2GB * 0.8
1512
DEFINE_Validator(tablet_meta_serialize_size_limit,
1513
                 [](const int64_t config) -> bool { return config < 1717986918; });
1514
1515
DEFINE_mInt64(pipeline_task_leakage_detect_period_secs, "60");
1516
DEFINE_mInt32(snappy_compression_block_size, "262144");
1517
DEFINE_mInt32(lz4_compression_block_size, "262144");
1518
1519
DEFINE_mBool(enable_pipeline_task_leakage_detect, "false");
1520
1521
DEFINE_mInt32(check_score_rounds_num, "1000");
1522
1523
DEFINE_Int32(query_cache_size, "512");
1524
1525
// Enable validation to check the correctness of table size.
1526
DEFINE_Bool(enable_table_size_correctness_check, "false");
1527
DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
1528
DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
1529
1530
// The number of compaction tasks generated each time.
1531
// -1 means automatic number, other values mean fixed number.
1532
DEFINE_mInt32(compaction_num_per_round, "-1");
1533
// Max automatic compaction task generated num per round.
1534
// Only valid if "compaction_num_per_round = 0"
1535
DEFINE_mInt32(max_automatic_compaction_num_per_round, "64");
1536
1537
DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300");
1538
DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10");
1539
DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true");
1540
DEFINE_mInt32(schema_dict_cache_capacity, "4096");
1541
1542
// whether to prune rows with delete sign = 1 in base compaction
1543
// ATTN: this config is only for test
1544
DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
1545
1546
DEFINE_mBool(enable_mow_verbose_log, "false");
1547
1548
DEFINE_mInt32(tablet_sched_delay_time_ms, "5000");
1549
DEFINE_mInt32(load_trigger_compaction_version_percent, "66");
1550
DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
1551
DEFINE_mBool(enable_compaction_pause_on_high_memory, "true");
1552
1553
DEFINE_mBool(enable_quorum_success_write, "true");
1554
DEFINE_mDouble(quorum_success_max_wait_multiplier, "0.2");
1555
DEFINE_mInt64(quorum_success_min_wait_seconds, "10");
1556
DEFINE_mInt32(quorum_success_remaining_timeout_seconds, "30");
1557
1558
DEFINE_mBool(enable_calc_delete_bitmap_between_segments_concurrently, "false");
1559
1560
DEFINE_mBool(enable_update_delete_bitmap_kv_check_core, "false");
1561
1562
// the max length of segments key bounds, in bytes
1563
// ATTENTION: as long as this conf has ever been enabled, cluster downgrade and backup recovery will no longer be supported.
1564
DEFINE_mInt32(segments_key_bounds_truncation_threshold, "-1");
1565
// ATTENTION: for test only, use random segments key bounds truncation threshold every time
1566
DEFINE_mBool(random_segments_key_bounds_truncation, "false");
1567
// p0, daily, rqg, external
1568
DEFINE_String(fuzzy_test_type, "");
1569
1570
DEFINE_mBool(enable_auto_clone_on_compaction_missing_version, "false");
1571
1572
DEFINE_mBool(enable_auto_clone_on_mow_publish_missing_version, "false");
1573
1574
// clang-format off
1575
#ifdef BE_TEST
1576
// test s3
1577
DEFINE_String(test_s3_resource, "resource");
1578
DEFINE_String(test_s3_ak, "ak");
1579
DEFINE_String(test_s3_sk, "sk");
1580
DEFINE_String(test_s3_endpoint, "endpoint");
1581
DEFINE_String(test_s3_region, "region");
1582
DEFINE_String(test_s3_bucket, "bucket");
1583
DEFINE_String(test_s3_prefix, "prefix");
1584
#endif
1585
// clang-format on
1586
1587
std::map<std::string, Register::Field>* Register::_s_field_map = nullptr;
1588
std::map<std::string, std::function<bool()>>* RegisterConfValidator::_s_field_validator = nullptr;
1589
std::map<std::string, std::string>* full_conf_map = nullptr;
1590
1591
std::mutex custom_conf_lock;
1592
1593
std::mutex mutable_string_config_lock;
1594
1595
// trim string
1596
8.21k
std::string& trim(std::string& s) {
1597
    // rtrim
1598
8.21k
    s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char c) { return !std::isspace(c); })
1599
8.21k
                    .base(),
1600
8.21k
            s.end());
1601
    // ltrim
1602
8.21k
    s.erase(s.begin(),
1603
8.21k
            std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isspace(c); }));
1604
8.21k
    return s;
1605
8.21k
}
1606
1607
// split string by '='
1608
472
void splitkv(const std::string& s, std::string& k, std::string& v) {
1609
472
    const char sep = '=';
1610
472
    int start = 0;
1611
472
    int end = 0;
1612
472
    if ((end = s.find(sep, start)) != std::string::npos) {
1613
472
        k = s.substr(start, end - start);
1614
472
        v = s.substr(end + 1);
1615
472
    } else {
1616
0
        k = s;
1617
0
        v = "";
1618
0
    }
1619
472
}
1620
1621
// replace env variables
1622
6.21k
bool replaceenv(std::string& s) {
1623
6.21k
    std::size_t pos = 0;
1624
6.21k
    std::size_t start = 0;
1625
6.29k
    while ((start = s.find("${", pos)) != std::string::npos) {
1626
79
        std::size_t end = s.find('}', start + 2);
1627
79
        if (end == std::string::npos) {
1628
0
            return false;
1629
0
        }
1630
79
        std::string envkey = s.substr(start + 2, end - start - 2);
1631
79
        const char* envval = std::getenv(envkey.c_str());
1632
79
        if (envval == nullptr) {
1633
0
            return false;
1634
0
        }
1635
79
        s.erase(start, end - start + 1);
1636
79
        s.insert(start, envval);
1637
79
        pos = start + strlen(envval);
1638
79
    }
1639
6.21k
    return true;
1640
6.21k
}
1641
1642
bool strtox(const std::string& valstr, bool& retval);
1643
bool strtox(const std::string& valstr, int16_t& retval);
1644
bool strtox(const std::string& valstr, int32_t& retval);
1645
bool strtox(const std::string& valstr, int64_t& retval);
1646
bool strtox(const std::string& valstr, double& retval);
1647
bool strtox(const std::string& valstr, std::string& retval);
1648
1649
template <typename T>
1650
15
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
15
    std::stringstream ss(valstr);
1652
15
    std::string item;
1653
15
    T t;
1654
34
    while (std::getline(ss, item, ',')) {
1655
19
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
19
        retval.push_back(t);
1659
19
    }
1660
15
    return true;
1661
15
}
_ZN5doris6config6strtoxIbEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIT_SaISB_EE
Line
Count
Source
1650
1
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
1
    std::stringstream ss(valstr);
1652
1
    std::string item;
1653
1
    T t;
1654
4
    while (std::getline(ss, item, ',')) {
1655
3
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
3
        retval.push_back(t);
1659
3
    }
1660
1
    return true;
1661
1
}
_ZN5doris6config6strtoxIsEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIT_SaISB_EE
Line
Count
Source
1650
1
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
1
    std::stringstream ss(valstr);
1652
1
    std::string item;
1653
1
    T t;
1654
4
    while (std::getline(ss, item, ',')) {
1655
3
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
3
        retval.push_back(t);
1659
3
    }
1660
1
    return true;
1661
1
}
_ZN5doris6config6strtoxIiEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIT_SaISB_EE
Line
Count
Source
1650
1
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
1
    std::stringstream ss(valstr);
1652
1
    std::string item;
1653
1
    T t;
1654
4
    while (std::getline(ss, item, ',')) {
1655
3
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
3
        retval.push_back(t);
1659
3
    }
1660
1
    return true;
1661
1
}
_ZN5doris6config6strtoxIlEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIT_SaISB_EE
Line
Count
Source
1650
1
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
1
    std::stringstream ss(valstr);
1652
1
    std::string item;
1653
1
    T t;
1654
4
    while (std::getline(ss, item, ',')) {
1655
3
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
3
        retval.push_back(t);
1659
3
    }
1660
1
    return true;
1661
1
}
_ZN5doris6config6strtoxIdEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIT_SaISB_EE
Line
Count
Source
1650
1
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
1
    std::stringstream ss(valstr);
1652
1
    std::string item;
1653
1
    T t;
1654
4
    while (std::getline(ss, item, ',')) {
1655
3
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
3
        retval.push_back(t);
1659
3
    }
1660
1
    return true;
1661
1
}
_ZN5doris6config6strtoxINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEbRKS7_RSt6vectorIT_SaISB_EE
Line
Count
Source
1650
10
bool strtox(const std::string& valstr, std::vector<T>& retval) {
1651
10
    std::stringstream ss(valstr);
1652
10
    std::string item;
1653
10
    T t;
1654
14
    while (std::getline(ss, item, ',')) {
1655
4
        if (!strtox(trim(item), t)) {
1656
0
            return false;
1657
0
        }
1658
4
        retval.push_back(t);
1659
4
    }
1660
10
    return true;
1661
10
}
1662
1663
1.40k
bool strtox(const std::string& valstr, bool& retval) {
1664
1.40k
    if (valstr == "true") {
1665
773
        retval = true;
1666
773
    } else if (valstr == "false") {
1667
627
        retval = false;
1668
627
    } else {
1669
1
        return false;
1670
1
    }
1671
1.40k
    return true;
1672
1.40k
}
1673
1674
template <typename T>
1675
4.12k
bool strtointeger(const std::string& valstr, T& retval) {
1676
4.12k
    if (valstr.length() == 0) {
1677
0
        return false; // empty-string is only allowed for string type.
1678
0
    }
1679
4.12k
    char* end;
1680
4.12k
    errno = 0;
1681
4.12k
    const char* valcstr = valstr.c_str();
1682
4.12k
    int64_t ret64 = strtoll(valcstr, &end, 10);
1683
4.12k
    if (errno || end != valcstr + strlen(valcstr)) {
1684
0
        return false; // bad parse
1685
0
    }
1686
4.12k
    T tmp = retval;
1687
4.12k
    retval = static_cast<T>(ret64);
1688
4.12k
    if (retval != ret64) {
1689
1
        retval = tmp;
1690
1
        return false;
1691
1
    }
1692
4.12k
    return true;
1693
4.12k
}
_ZN5doris6config12strtointegerIsEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1675
27
bool strtointeger(const std::string& valstr, T& retval) {
1676
27
    if (valstr.length() == 0) {
1677
0
        return false; // empty-string is only allowed for string type.
1678
0
    }
1679
27
    char* end;
1680
27
    errno = 0;
1681
27
    const char* valcstr = valstr.c_str();
1682
27
    int64_t ret64 = strtoll(valcstr, &end, 10);
1683
27
    if (errno || end != valcstr + strlen(valcstr)) {
1684
0
        return false; // bad parse
1685
0
    }
1686
27
    T tmp = retval;
1687
27
    retval = static_cast<T>(ret64);
1688
27
    if (retval != ret64) {
1689
0
        retval = tmp;
1690
0
        return false;
1691
0
    }
1692
27
    return true;
1693
27
}
_ZN5doris6config12strtointegerIiEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1675
3.00k
bool strtointeger(const std::string& valstr, T& retval) {
1676
3.00k
    if (valstr.length() == 0) {
1677
0
        return false; // empty-string is only allowed for string type.
1678
0
    }
1679
3.00k
    char* end;
1680
3.00k
    errno = 0;
1681
3.00k
    const char* valcstr = valstr.c_str();
1682
3.00k
    int64_t ret64 = strtoll(valcstr, &end, 10);
1683
3.00k
    if (errno || end != valcstr + strlen(valcstr)) {
1684
0
        return false; // bad parse
1685
0
    }
1686
3.00k
    T tmp = retval;
1687
3.00k
    retval = static_cast<T>(ret64);
1688
3.00k
    if (retval != ret64) {
1689
1
        retval = tmp;
1690
1
        return false;
1691
1
    }
1692
3.00k
    return true;
1693
3.00k
}
_ZN5doris6config12strtointegerIlEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1675
1.09k
bool strtointeger(const std::string& valstr, T& retval) {
1676
1.09k
    if (valstr.length() == 0) {
1677
0
        return false; // empty-string is only allowed for string type.
1678
0
    }
1679
1.09k
    char* end;
1680
1.09k
    errno = 0;
1681
1.09k
    const char* valcstr = valstr.c_str();
1682
1.09k
    int64_t ret64 = strtoll(valcstr, &end, 10);
1683
1.09k
    if (errno || end != valcstr + strlen(valcstr)) {
1684
0
        return false; // bad parse
1685
0
    }
1686
1.09k
    T tmp = retval;
1687
1.09k
    retval = static_cast<T>(ret64);
1688
1.09k
    if (retval != ret64) {
1689
0
        retval = tmp;
1690
0
        return false;
1691
0
    }
1692
1.09k
    return true;
1693
1.09k
}
1694
1695
27
bool strtox(const std::string& valstr, int16_t& retval) {
1696
27
    return strtointeger(valstr, retval);
1697
27
}
1698
1699
3.00k
bool strtox(const std::string& valstr, int32_t& retval) {
1700
3.00k
    return strtointeger(valstr, retval);
1701
3.00k
}
1702
1703
1.09k
bool strtox(const std::string& valstr, int64_t& retval) {
1704
1.09k
    return strtointeger(valstr, retval);
1705
1.09k
}
1706
1707
160
bool strtox(const std::string& valstr, double& retval) {
1708
160
    if (valstr.length() == 0) {
1709
1
        return false; // empty-string is only allowed for string type.
1710
1
    }
1711
159
    char* end = nullptr;
1712
159
    errno = 0;
1713
159
    const char* valcstr = valstr.c_str();
1714
159
    retval = strtod(valcstr, &end);
1715
159
    if (errno || end != valcstr + strlen(valcstr)) {
1716
0
        return false; // bad parse
1717
0
    }
1718
159
    return true;
1719
159
}
1720
1721
537
bool strtox(const std::string& valstr, std::string& retval) {
1722
537
    retval = valstr;
1723
537
    return true;
1724
537
}
1725
1726
template <typename T>
1727
6.21k
bool convert(const std::string& value, T& retval) {
1728
6.21k
    std::string valstr(value);
1729
6.21k
    trim(valstr);
1730
6.21k
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
6.21k
    return strtox(valstr, retval);
1734
6.21k
}
_ZN5doris6config7convertISt6vectorIbSaIbEEEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1
bool convert(const std::string& value, T& retval) {
1728
1
    std::string valstr(value);
1729
1
    trim(valstr);
1730
1
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1
    return strtox(valstr, retval);
1734
1
}
_ZN5doris6config7convertISt6vectorIsSaIsEEEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1
bool convert(const std::string& value, T& retval) {
1728
1
    std::string valstr(value);
1729
1
    trim(valstr);
1730
1
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1
    return strtox(valstr, retval);
1734
1
}
_ZN5doris6config7convertISt6vectorIiSaIiEEEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1
bool convert(const std::string& value, T& retval) {
1728
1
    std::string valstr(value);
1729
1
    trim(valstr);
1730
1
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1
    return strtox(valstr, retval);
1734
1
}
_ZN5doris6config7convertISt6vectorIlSaIlEEEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1
bool convert(const std::string& value, T& retval) {
1728
1
    std::string valstr(value);
1729
1
    trim(valstr);
1730
1
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1
    return strtox(valstr, retval);
1734
1
}
_ZN5doris6config7convertISt6vectorIdSaIdEEEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1
bool convert(const std::string& value, T& retval) {
1728
1
    std::string valstr(value);
1729
1
    trim(valstr);
1730
1
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1
    return strtox(valstr, retval);
1734
1
}
_ZN5doris6config7convertISt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS8_EEEEbRKS8_RT_
Line
Count
Source
1727
10
bool convert(const std::string& value, T& retval) {
1728
10
    std::string valstr(value);
1729
10
    trim(valstr);
1730
10
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
10
    return strtox(valstr, retval);
1734
10
}
_ZN5doris6config7convertIbEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1.39k
bool convert(const std::string& value, T& retval) {
1728
1.39k
    std::string valstr(value);
1729
1.39k
    trim(valstr);
1730
1.39k
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1.39k
    return strtox(valstr, retval);
1734
1.39k
}
_ZN5doris6config7convertIsEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
24
bool convert(const std::string& value, T& retval) {
1728
24
    std::string valstr(value);
1729
24
    trim(valstr);
1730
24
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
24
    return strtox(valstr, retval);
1734
24
}
_ZN5doris6config7convertIiEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
3.00k
bool convert(const std::string& value, T& retval) {
1728
3.00k
    std::string valstr(value);
1729
3.00k
    trim(valstr);
1730
3.00k
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
3.00k
    return strtox(valstr, retval);
1734
3.00k
}
_ZN5doris6config7convertIlEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
1.08k
bool convert(const std::string& value, T& retval) {
1728
1.08k
    std::string valstr(value);
1729
1.08k
    trim(valstr);
1730
1.08k
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
1.08k
    return strtox(valstr, retval);
1734
1.08k
}
_ZN5doris6config7convertIdEEbRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERT_
Line
Count
Source
1727
157
bool convert(const std::string& value, T& retval) {
1728
157
    std::string valstr(value);
1729
157
    trim(valstr);
1730
157
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
157
    return strtox(valstr, retval);
1734
157
}
_ZN5doris6config7convertINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEbRKS7_RT_
Line
Count
Source
1727
533
bool convert(const std::string& value, T& retval) {
1728
533
    std::string valstr(value);
1729
533
    trim(valstr);
1730
533
    if (!replaceenv(valstr)) {
1731
0
        return false;
1732
0
    }
1733
533
    return strtox(valstr, retval);
1734
533
}
1735
1736
// load conf file
1737
24
bool Properties::load(const char* conf_file, bool must_exist) {
1738
    // if conf_file is null, use the empty props
1739
24
    if (conf_file == nullptr) {
1740
2
        return true;
1741
2
    }
1742
1743
    // open the conf file
1744
22
    std::ifstream input(conf_file);
1745
22
    if (!input.is_open()) {
1746
7
        if (must_exist) {
1747
0
            std::cerr << "config::load() failed to open the file:" << conf_file << std::endl;
1748
0
            return false;
1749
0
        }
1750
7
        return true;
1751
7
    }
1752
1753
    // load properties
1754
15
    std::string line;
1755
15
    std::string key;
1756
15
    std::string value;
1757
15
    line.reserve(512);
1758
1.05k
    while (input) {
1759
        // read one line at a time
1760
1.03k
        std::getline(input, line);
1761
1762
        // remove left and right spaces
1763
1.03k
        trim(line);
1764
1765
        // ignore comments
1766
1.03k
        if (line.empty() || line[0] == '#') {
1767
565
            continue;
1768
565
        }
1769
1770
        // read key and value
1771
472
        splitkv(line, key, value);
1772
472
        trim(key);
1773
472
        trim(value);
1774
1775
        // insert into file_conf_map
1776
472
        file_conf_map[key] = value;
1777
472
    }
1778
1779
    // close the conf file
1780
15
    input.close();
1781
1782
15
    return true;
1783
22
}
1784
1785
template <typename T>
1786
bool Properties::get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set,
1787
11.4k
                                std::string& rawval) const {
1788
11.4k
    const auto& it = file_conf_map.find(std::string(key));
1789
11.4k
    std::string valstr;
1790
11.4k
    if (it == file_conf_map.end()) {
1791
11.1k
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
5.30k
            *is_retval_set = false;
1794
5.30k
            return true;
1795
5.83k
        } else {
1796
5.83k
            valstr = std::string(defstr);
1797
5.83k
        }
1798
11.1k
    } else {
1799
294
        valstr = it->second;
1800
294
    }
1801
6.12k
    rawval = valstr;
1802
6.12k
    *is_retval_set = true;
1803
6.12k
    return convert(valstr, retval);
1804
11.4k
}
_ZNK5doris6config10Properties14get_or_defaultIbEEbPKcS4_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
2.46k
                                std::string& rawval) const {
1788
2.46k
    const auto& it = file_conf_map.find(std::string(key));
1789
2.46k
    std::string valstr;
1790
2.46k
    if (it == file_conf_map.end()) {
1791
2.36k
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
1.12k
            *is_retval_set = false;
1794
1.12k
            return true;
1795
1.24k
        } else {
1796
1.24k
            valstr = std::string(defstr);
1797
1.24k
        }
1798
2.36k
    } else {
1799
101
        valstr = it->second;
1800
101
    }
1801
1.34k
    rawval = valstr;
1802
1.34k
    *is_retval_set = true;
1803
1.34k
    return convert(valstr, retval);
1804
2.46k
}
_ZNK5doris6config10Properties14get_or_defaultIsEEbPKcS4_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
36
                                std::string& rawval) const {
1788
36
    const auto& it = file_conf_map.find(std::string(key));
1789
36
    std::string valstr;
1790
36
    if (it == file_conf_map.end()) {
1791
36
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
16
            *is_retval_set = false;
1794
16
            return true;
1795
20
        } else {
1796
20
            valstr = std::string(defstr);
1797
20
        }
1798
36
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
20
    rawval = valstr;
1802
20
    *is_retval_set = true;
1803
20
    return convert(valstr, retval);
1804
36
}
_ZNK5doris6config10Properties14get_or_defaultIiEEbPKcS4_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
5.61k
                                std::string& rawval) const {
1788
5.61k
    const auto& it = file_conf_map.find(std::string(key));
1789
5.61k
    std::string valstr;
1790
5.61k
    if (it == file_conf_map.end()) {
1791
5.49k
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
2.62k
            *is_retval_set = false;
1794
2.62k
            return true;
1795
2.87k
        } else {
1796
2.87k
            valstr = std::string(defstr);
1797
2.87k
        }
1798
5.49k
    } else {
1799
115
        valstr = it->second;
1800
115
    }
1801
2.98k
    rawval = valstr;
1802
2.98k
    *is_retval_set = true;
1803
2.98k
    return convert(valstr, retval);
1804
5.61k
}
_ZNK5doris6config10Properties14get_or_defaultIlEEbPKcS4_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
2.02k
                                std::string& rawval) const {
1788
2.02k
    const auto& it = file_conf_map.find(std::string(key));
1789
2.02k
    std::string valstr;
1790
2.02k
    if (it == file_conf_map.end()) {
1791
2.00k
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
948
            *is_retval_set = false;
1794
948
            return true;
1795
1.06k
        } else {
1796
1.06k
            valstr = std::string(defstr);
1797
1.06k
        }
1798
2.00k
    } else {
1799
16
        valstr = it->second;
1800
16
    }
1801
1.07k
    rawval = valstr;
1802
1.07k
    *is_retval_set = true;
1803
1.07k
    return convert(valstr, retval);
1804
2.02k
}
_ZNK5doris6config10Properties14get_or_defaultIdEEbPKcS4_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
291
                                std::string& rawval) const {
1788
291
    const auto& it = file_conf_map.find(std::string(key));
1789
291
    std::string valstr;
1790
291
    if (it == file_conf_map.end()) {
1791
291
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
136
            *is_retval_set = false;
1794
136
            return true;
1795
155
        } else {
1796
155
            valstr = std::string(defstr);
1797
155
        }
1798
291
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
155
    rawval = valstr;
1802
155
    *is_retval_set = true;
1803
155
    return convert(valstr, retval);
1804
291
}
_ZNK5doris6config10Properties14get_or_defaultINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEbPKcSA_RT_PbRS8_
Line
Count
Source
1787
978
                                std::string& rawval) const {
1788
978
    const auto& it = file_conf_map.find(std::string(key));
1789
978
    std::string valstr;
1790
978
    if (it == file_conf_map.end()) {
1791
916
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
448
            *is_retval_set = false;
1794
448
            return true;
1795
468
        } else {
1796
468
            valstr = std::string(defstr);
1797
468
        }
1798
916
    } else {
1799
62
        valstr = it->second;
1800
62
    }
1801
530
    rawval = valstr;
1802
530
    *is_retval_set = true;
1803
530
    return convert(valstr, retval);
1804
978
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorIbSaIbEEEEbPKcS7_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
1
                                std::string& rawval) const {
1788
1
    const auto& it = file_conf_map.find(std::string(key));
1789
1
    std::string valstr;
1790
1
    if (it == file_conf_map.end()) {
1791
1
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
0
            *is_retval_set = false;
1794
0
            return true;
1795
1
        } else {
1796
1
            valstr = std::string(defstr);
1797
1
        }
1798
1
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
1
    rawval = valstr;
1802
1
    *is_retval_set = true;
1803
1
    return convert(valstr, retval);
1804
1
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorIsSaIsEEEEbPKcS7_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
1
                                std::string& rawval) const {
1788
1
    const auto& it = file_conf_map.find(std::string(key));
1789
1
    std::string valstr;
1790
1
    if (it == file_conf_map.end()) {
1791
1
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
0
            *is_retval_set = false;
1794
0
            return true;
1795
1
        } else {
1796
1
            valstr = std::string(defstr);
1797
1
        }
1798
1
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
1
    rawval = valstr;
1802
1
    *is_retval_set = true;
1803
1
    return convert(valstr, retval);
1804
1
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorIiSaIiEEEEbPKcS7_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
1
                                std::string& rawval) const {
1788
1
    const auto& it = file_conf_map.find(std::string(key));
1789
1
    std::string valstr;
1790
1
    if (it == file_conf_map.end()) {
1791
1
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
0
            *is_retval_set = false;
1794
0
            return true;
1795
1
        } else {
1796
1
            valstr = std::string(defstr);
1797
1
        }
1798
1
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
1
    rawval = valstr;
1802
1
    *is_retval_set = true;
1803
1
    return convert(valstr, retval);
1804
1
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorIlSaIlEEEEbPKcS7_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
1
                                std::string& rawval) const {
1788
1
    const auto& it = file_conf_map.find(std::string(key));
1789
1
    std::string valstr;
1790
1
    if (it == file_conf_map.end()) {
1791
1
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
0
            *is_retval_set = false;
1794
0
            return true;
1795
1
        } else {
1796
1
            valstr = std::string(defstr);
1797
1
        }
1798
1
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
1
    rawval = valstr;
1802
1
    *is_retval_set = true;
1803
1
    return convert(valstr, retval);
1804
1
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorIdSaIdEEEEbPKcS7_RT_PbRNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
Line
Count
Source
1787
1
                                std::string& rawval) const {
1788
1
    const auto& it = file_conf_map.find(std::string(key));
1789
1
    std::string valstr;
1790
1
    if (it == file_conf_map.end()) {
1791
1
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
0
            *is_retval_set = false;
1794
0
            return true;
1795
1
        } else {
1796
1
            valstr = std::string(defstr);
1797
1
        }
1798
1
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
1
    rawval = valstr;
1802
1
    *is_retval_set = true;
1803
1
    return convert(valstr, retval);
1804
1
}
_ZNK5doris6config10Properties14get_or_defaultISt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS9_EEEEbPKcSD_RT_PbRS9_
Line
Count
Source
1787
18
                                std::string& rawval) const {
1788
18
    const auto& it = file_conf_map.find(std::string(key));
1789
18
    std::string valstr;
1790
18
    if (it == file_conf_map.end()) {
1791
18
        if (defstr == nullptr) {
1792
            // Not found in conf map, and no default value need to be set, just return
1793
8
            *is_retval_set = false;
1794
8
            return true;
1795
10
        } else {
1796
10
            valstr = std::string(defstr);
1797
10
        }
1798
18
    } else {
1799
0
        valstr = it->second;
1800
0
    }
1801
10
    rawval = valstr;
1802
10
    *is_retval_set = true;
1803
10
    return convert(valstr, retval);
1804
18
}
1805
1806
0
void Properties::set(const std::string& key, const std::string& val) {
1807
0
    file_conf_map.emplace(key, val);
1808
0
}
1809
1810
5
void Properties::set_force(const std::string& key, const std::string& val) {
1811
5
    file_conf_map[key] = val;
1812
5
}
1813
1814
5
Status Properties::dump(const std::string& conffile) {
1815
5
    std::string conffile_tmp = conffile + ".tmp";
1816
5
    io::FileWriterPtr file_writer;
1817
5
    RETURN_IF_ERROR(io::global_local_filesystem()->create_file(conffile_tmp, &file_writer));
1818
5
    RETURN_IF_ERROR(file_writer->append("# THIS IS AN AUTO GENERATED CONFIG FILE.\n"));
1819
5
    RETURN_IF_ERROR(file_writer->append(
1820
5
            "# You can modify this file manually, and the configurations in this file\n"));
1821
5
    RETURN_IF_ERROR(file_writer->append("# will overwrite the configurations in be.conf\n\n"));
1822
1823
85
    for (auto const& iter : file_conf_map) {
1824
85
        RETURN_IF_ERROR(file_writer->append(iter.first));
1825
85
        RETURN_IF_ERROR(file_writer->append(" = "));
1826
85
        RETURN_IF_ERROR(file_writer->append(iter.second));
1827
85
        RETURN_IF_ERROR(file_writer->append("\n"));
1828
85
    }
1829
1830
5
    RETURN_IF_ERROR(file_writer->close());
1831
1832
5
    return io::global_local_filesystem()->rename(conffile_tmp, conffile);
1833
5
}
1834
1835
template <typename T>
1836
14
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
14
    size_t last = v.size() - 1;
1838
33
    for (size_t i = 0; i < v.size(); ++i) {
1839
19
        out << v[i];
1840
19
        if (i != last) {
1841
13
            out << ", ";
1842
13
        }
1843
19
    }
1844
14
    return out;
1845
14
}
_ZN5doris6configlsIbEERSoS2_RKSt6vectorIT_SaIS4_EE
Line
Count
Source
1836
1
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
1
    size_t last = v.size() - 1;
1838
4
    for (size_t i = 0; i < v.size(); ++i) {
1839
3
        out << v[i];
1840
3
        if (i != last) {
1841
2
            out << ", ";
1842
2
        }
1843
3
    }
1844
1
    return out;
1845
1
}
_ZN5doris6configlsIsEERSoS2_RKSt6vectorIT_SaIS4_EE
Line
Count
Source
1836
1
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
1
    size_t last = v.size() - 1;
1838
4
    for (size_t i = 0; i < v.size(); ++i) {
1839
3
        out << v[i];
1840
3
        if (i != last) {
1841
2
            out << ", ";
1842
2
        }
1843
3
    }
1844
1
    return out;
1845
1
}
_ZN5doris6configlsIiEERSoS2_RKSt6vectorIT_SaIS4_EE
Line
Count
Source
1836
1
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
1
    size_t last = v.size() - 1;
1838
4
    for (size_t i = 0; i < v.size(); ++i) {
1839
3
        out << v[i];
1840
3
        if (i != last) {
1841
2
            out << ", ";
1842
2
        }
1843
3
    }
1844
1
    return out;
1845
1
}
_ZN5doris6configlsIlEERSoS2_RKSt6vectorIT_SaIS4_EE
Line
Count
Source
1836
1
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
1
    size_t last = v.size() - 1;
1838
4
    for (size_t i = 0; i < v.size(); ++i) {
1839
3
        out << v[i];
1840
3
        if (i != last) {
1841
2
            out << ", ";
1842
2
        }
1843
3
    }
1844
1
    return out;
1845
1
}
_ZN5doris6configlsIdEERSoS2_RKSt6vectorIT_SaIS4_EE
Line
Count
Source
1836
1
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
1
    size_t last = v.size() - 1;
1838
4
    for (size_t i = 0; i < v.size(); ++i) {
1839
3
        out << v[i];
1840
3
        if (i != last) {
1841
2
            out << ", ";
1842
2
        }
1843
3
    }
1844
1
    return out;
1845
1
}
_ZN5doris6configlsINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEERSoS8_RKSt6vectorIT_SaISA_EE
Line
Count
Source
1836
9
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
1837
9
    size_t last = v.size() - 1;
1838
13
    for (size_t i = 0; i < v.size(); ++i) {
1839
4
        out << v[i];
1840
4
        if (i != last) {
1841
3
            out << ", ";
1842
3
        }
1843
4
    }
1844
9
    return out;
1845
9
}
1846
1847
#define SET_FIELD(FIELD, TYPE, FILL_CONF_MAP, SET_TO_DEFAULT)                                  \
1848
35.0k
    if (strcmp((FIELD).type, #TYPE) == 0) {                                                    \
1849
11.4k
        TYPE new_value = TYPE();                                                               \
1850
11.4k
        bool is_newval_set = false;                                                            \
1851
11.4k
        std::string raw_value;                                                                 \
1852
11.4k
        if (!props.get_or_default((FIELD).name, ((SET_TO_DEFAULT) ? (FIELD).defval : nullptr), \
1853
11.4k
                                  new_value, &is_newval_set, raw_value)) {                     \
1854
0
            std::cerr << "config field error: " << (FIELD).name << " = \"" << raw_value << '"' \
1855
0
                      << std::endl;                                                            \
1856
0
            return false;                                                                      \
1857
0
        }                                                                                      \
1858
11.4k
        if (!is_newval_set) {                                                                  \
1859
5.30k
            continue;                                                                          \
1860
5.30k
        }                                                                                      \
1861
11.4k
        TYPE& ref_conf_value = *reinterpret_cast<TYPE*>((FIELD).storage);                      \
1862
6.12k
        TYPE old_value = ref_conf_value;                                                       \
1863
6.12k
        ref_conf_value = new_value;                                                            \
1864
6.12k
        if (RegisterConfValidator::_s_field_validator != nullptr) {                            \
1865
6.12k
            auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name);    \
1866
6.12k
            if (validator != RegisterConfValidator::_s_field_validator->end() &&               \
1867
6.12k
                !(validator->second)()) {                                                      \
1868
0
                ref_conf_value = old_value;                                                    \
1869
0
                std::cerr << "validate " << (FIELD).name << "=" << new_value << " failed"      \
1870
0
                          << std::endl;                                                        \
1871
0
                return false;                                                                  \
1872
0
            }                                                                                  \
1873
6.12k
        }                                                                                      \
1874
6.12k
        if (FILL_CONF_MAP) {                                                                   \
1875
5.44k
            std::ostringstream oss;                                                            \
1876
5.44k
            oss << ref_conf_value;                                                             \
1877
5.44k
            (*full_conf_map)[(FIELD).name] = oss.str();                                        \
1878
5.44k
        }                                                                                      \
1879
6.12k
        continue;                                                                              \
1880
6.12k
    }
1881
1882
// init conf fields
1883
19
bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_to_default) {
1884
19
    Properties props;
1885
    // load properties file
1886
19
    if (!props.load(conf_file, must_exist)) {
1887
0
        return false;
1888
0
    }
1889
    // fill full_conf_map ?
1890
19
    if (fill_conf_map && full_conf_map == nullptr) {
1891
9
        full_conf_map = new std::map<std::string, std::string>();
1892
9
    }
1893
1894
    // set conf fields
1895
11.4k
    for (const auto& it : *Register::_s_field_map) {
1896
11.4k
        SET_FIELD(it.second, bool, fill_conf_map, set_to_default);
1897
8.96k
        SET_FIELD(it.second, int16_t, fill_conf_map, set_to_default);
1898
8.92k
        SET_FIELD(it.second, int32_t, fill_conf_map, set_to_default);
1899
3.31k
        SET_FIELD(it.second, int64_t, fill_conf_map, set_to_default);
1900
1.29k
        SET_FIELD(it.second, double, fill_conf_map, set_to_default);
1901
1.00k
        SET_FIELD(it.second, std::string, fill_conf_map, set_to_default);
1902
23
        SET_FIELD(it.second, std::vector<bool>, fill_conf_map, set_to_default);
1903
22
        SET_FIELD(it.second, std::vector<int16_t>, fill_conf_map, set_to_default);
1904
21
        SET_FIELD(it.second, std::vector<int32_t>, fill_conf_map, set_to_default);
1905
20
        SET_FIELD(it.second, std::vector<int64_t>, fill_conf_map, set_to_default);
1906
19
        SET_FIELD(it.second, std::vector<double>, fill_conf_map, set_to_default);
1907
18
        SET_FIELD(it.second, std::vector<std::string>, fill_conf_map, set_to_default);
1908
0
    }
1909
1910
19
    if (config::is_cloud_mode()) {
1911
2
        auto st = config::set_config("enable_file_cache", "true", true, true);
1912
2
        LOG(INFO) << "set config enable_file_cache "
1913
2
                  << "true"
1914
2
                  << " " << st;
1915
2
    }
1916
1917
19
    return true;
1918
19
}
1919
1920
#define UPDATE_FIELD(FIELD, VALUE, TYPE, PERSIST)                                                  \
1921
185
    if (strcmp((FIELD).type, #TYPE) == 0) {                                                        \
1922
93
        TYPE new_value;                                                                            \
1923
93
        if (!convert((VALUE), new_value)) {                                                        \
1924
3
            return Status::Error<ErrorCode::INVALID_ARGUMENT, false>("convert '{}' as {} failed",  \
1925
3
                                                                     VALUE, #TYPE);                \
1926
3
        }                                                                                          \
1927
93
        TYPE& ref_conf_value = *reinterpret_cast<TYPE*>((FIELD).storage);                          \
1928
90
        TYPE old_value = ref_conf_value;                                                           \
1929
90
        if (RegisterConfValidator::_s_field_validator != nullptr) {                                \
1930
90
            auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name);        \
1931
90
            if (validator != RegisterConfValidator::_s_field_validator->end() &&                   \
1932
90
                !(validator->second)()) {                                                          \
1933
0
                ref_conf_value = old_value;                                                        \
1934
0
                return Status::Error<ErrorCode::INVALID_ARGUMENT, false>("validate {}={} failed",  \
1935
0
                                                                         (FIELD).name, new_value); \
1936
0
            }                                                                                      \
1937
90
        }                                                                                          \
1938
90
        ref_conf_value = new_value;                                                                \
1939
90
        if (full_conf_map != nullptr) {                                                            \
1940
89
            std::ostringstream oss;                                                                \
1941
89
            oss << new_value;                                                                      \
1942
89
            (*full_conf_map)[(FIELD).name] = oss.str();                                            \
1943
89
        }                                                                                          \
1944
90
        if (PERSIST) {                                                                             \
1945
5
            RETURN_IF_ERROR(persist_config(std::string((FIELD).name), VALUE));                     \
1946
5
        }                                                                                          \
1947
90
        update_config(std::string((FIELD).name), VALUE);                                           \
1948
90
        return Status::OK();                                                                       \
1949
90
    }
1950
1951
// write config to be_custom.conf
1952
// the caller need to make sure that the given config is valid
1953
5
Status persist_config(const std::string& field, const std::string& value) {
1954
    // lock to make sure only one thread can modify the be_custom.conf
1955
5
    std::lock_guard<std::mutex> l(custom_conf_lock);
1956
1957
5
    static const std::string conffile = config::custom_config_dir + "/be_custom.conf";
1958
1959
5
    Properties tmp_props;
1960
5
    if (!tmp_props.load(conffile.c_str(), false)) {
1961
0
        LOG(WARNING) << "failed to load " << conffile;
1962
0
        return Status::InternalError("failed to load conf file: {}", conffile);
1963
0
    }
1964
1965
5
    tmp_props.set_force(field, value);
1966
5
    return tmp_props.dump(conffile);
1967
5
}
1968
1969
Status set_config(const std::string& field, const std::string& value, bool need_persist,
1970
98
                  bool force) {
1971
98
    auto it = Register::_s_field_map->find(field);
1972
98
    if (it == Register::_s_field_map->end()) {
1973
1
        return Status::Error<ErrorCode::NOT_FOUND, false>("'{}' is not found", field);
1974
1
    }
1975
1976
97
    if (!force && !it->second.valmutable) {
1977
4
        return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR, false>(
1978
4
                "'{}' is not support to modify", field);
1979
4
    }
1980
1981
93
    UPDATE_FIELD(it->second, value, bool, need_persist);
1982
36
    UPDATE_FIELD(it->second, value, int16_t, need_persist);
1983
32
    UPDATE_FIELD(it->second, value, int32_t, need_persist);
1984
16
    UPDATE_FIELD(it->second, value, int64_t, need_persist);
1985
5
    UPDATE_FIELD(it->second, value, double, need_persist);
1986
3
    {
1987
        // add lock to ensure thread safe
1988
3
        std::lock_guard<std::mutex> lock(mutable_string_config_lock);
1989
3
        UPDATE_FIELD(it->second, value, std::string, need_persist);
1990
0
    }
1991
1992
    // The other types are not thread safe to change dynamically.
1993
0
    return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR, false>(
1994
0
            "'{}' is type of '{}' which is not support to modify", field, it->second.type);
1995
3
}
1996
1997
90
void update_config(const std::string& field, const std::string& value) {
1998
90
    if ("sys_log_level" == field) {
1999
        // update log level
2000
0
        update_logging(field, value);
2001
0
    }
2002
90
}
2003
2004
6
Status set_fuzzy_configs() {
2005
6
    std::unordered_map<std::string, std::string> fuzzy_field_and_value;
2006
6
    std::shared_ptr<std::mt19937_64> generator(new std::mt19937_64());
2007
6
    generator->seed(std::random_device()());
2008
6
    std::uniform_int_distribution<int64_t> distribution(0, 100);
2009
2010
    // if have set enable_fuzzy_mode=true in be.conf, will fuzzy those field and values
2011
6
    fuzzy_field_and_value["disable_storage_page_cache"] =
2012
6
            ((distribution(*generator) % 2) == 0) ? "true" : "false";
2013
6
    fuzzy_field_and_value["disable_segment_cache"] =
2014
6
            ((distribution(*generator) % 2) == 0) ? "true" : "false";
2015
6
    fuzzy_field_and_value["enable_system_metrics"] =
2016
6
            ((distribution(*generator) % 2) == 0) ? "true" : "false";
2017
6
    fuzzy_field_and_value["enable_set_in_bitmap_value"] =
2018
6
            ((distribution(*generator) % 2) == 0) ? "true" : "false";
2019
6
    fuzzy_field_and_value["enable_shrink_memory"] =
2020
6
            ((distribution(*generator) % 2) == 0) ? "true" : "false";
2021
6
    fuzzy_field_and_value["string_overflow_size"] =
2022
6
            ((distribution(*generator) % 2) == 0) ? "10" : "4294967295";
2023
2024
6
    std::uniform_int_distribution<int64_t> distribution2(-2, 10);
2025
6
    fuzzy_field_and_value["segments_key_bounds_truncation_threshold"] =
2026
6
            std::to_string(distribution2(*generator));
2027
2028
    // external
2029
6
    if (config::fuzzy_test_type == "external") {
2030
0
        std::uniform_int_distribution<int64_t> distribution3(0, 2);
2031
2032
0
        int64_t idx = distribution3(*generator);
2033
0
        fuzzy_field_and_value["max_hdfs_file_handle_cache_num"] =
2034
0
                (idx == 0) ? "0" : ((idx == 1) ? "10" : "20000");
2035
2036
0
        idx = distribution3(*generator);
2037
0
        fuzzy_field_and_value["max_hdfs_file_handle_cache_time_sec"] =
2038
0
                (idx == 0) ? "1" : ((idx == 1) ? "10" : "28800");
2039
2040
0
        idx = distribution3(*generator);
2041
0
        fuzzy_field_and_value["max_external_file_meta_cache_num"] =
2042
0
                (idx == 0) ? "0" : ((idx == 1) ? "10" : "1000");
2043
2044
0
        idx = distribution3(*generator);
2045
0
        fuzzy_field_and_value["common_obj_lru_cache_stale_sweep_time_sec"] =
2046
0
                (idx == 0) ? "0" : ((idx == 1) ? "10" : "900");
2047
2048
0
        idx = distribution3(*generator);
2049
0
        fuzzy_field_and_value["max_amplified_read_ratio"] =
2050
0
                (idx == 0) ? "0.1" : ((idx == 1) ? "0.8" : "1");
2051
2052
0
        idx = distribution3(*generator);
2053
0
        fuzzy_field_and_value["merged_oss_min_io_size"] =
2054
0
                (idx == 0) ? "4096" : ((idx == 1) ? "8192" : "1048576");
2055
2056
0
        idx = distribution3(*generator);
2057
0
        fuzzy_field_and_value["merged_hdfs_min_io_size"] =
2058
0
                (idx == 0) ? "4096" : ((idx == 1) ? "8192" : "1048576");
2059
0
    }
2060
2061
6
    fmt::memory_buffer buf;
2062
42
    for (auto& it : fuzzy_field_and_value) {
2063
42
        const auto& field = it.first;
2064
42
        const auto& value = it.second;
2065
42
        RETURN_IF_ERROR(set_config(field, value, false, true));
2066
42
        fmt::format_to(buf, "{}={}, ", field, value);
2067
42
    }
2068
6
    LOG(INFO) << fmt::format("FUZZY MODE IN BE: those variables have been changed: ({}).",
2069
6
                             fmt::to_string(buf));
2070
6
    return Status::OK();
2071
6
}
2072
2073
0
std::mutex* get_mutable_string_config_lock() {
2074
0
    return &mutable_string_config_lock;
2075
0
}
2076
2077
86
std::vector<std::vector<std::string>> get_config_info() {
2078
86
    std::vector<std::vector<std::string>> configs;
2079
86
    std::lock_guard<std::mutex> lock(mutable_string_config_lock);
2080
57.7k
    for (const auto& it : *full_conf_map) {
2081
57.7k
        auto field_it = Register::_s_field_map->find(it.first);
2082
57.7k
        if (field_it == Register::_s_field_map->end()) {
2083
0
            continue;
2084
0
        }
2085
2086
57.7k
        std::vector<std::string> _config;
2087
57.7k
        _config.push_back(it.first);
2088
2089
57.7k
        std::string config_val = it.second;
2090
        // For compatibility, this PR #32933 change the log dir's config logic,
2091
        // and deprecate the `sys_log_dir` config.
2092
57.7k
        if (it.first == "sys_log_dir" && config_val == "") {
2093
86
            config_val = fmt::format("{}/log", std::getenv("DORIS_HOME"));
2094
86
        }
2095
2096
57.7k
        _config.emplace_back(field_it->second.type);
2097
57.7k
        if (0 == strcmp(field_it->second.type, "bool")) {
2098
12.4k
            _config.emplace_back(config_val == "1" ? "true" : "false");
2099
45.2k
        } else {
2100
45.2k
            _config.push_back(config_val);
2101
45.2k
        }
2102
57.7k
        _config.emplace_back(field_it->second.valmutable ? "true" : "false");
2103
2104
57.7k
        configs.push_back(_config);
2105
57.7k
    }
2106
86
    return configs;
2107
86
}
2108
2109
} // namespace doris::config