Coverage Report

Created: 2025-04-12 13:58

/root/doris/be/src/common/config.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <cstdint>
21
#include <functional>
22
#include <map>
23
#include <mutex>
24
#include <string>
25
#include <utility>
26
#include <vector>
27
28
#define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME
29
30
#define DECLARE_Bool(name) DECLARE_FIELD(bool, name)
31
#define DECLARE_Int16(name) DECLARE_FIELD(int16_t, name)
32
#define DECLARE_Int32(name) DECLARE_FIELD(int32_t, name)
33
#define DECLARE_Int64(name) DECLARE_FIELD(int64_t, name)
34
#define DECLARE_Double(name) DECLARE_FIELD(double, name)
35
#define DECLARE_String(name) DECLARE_FIELD(std::string, name)
36
#define DECLARE_Bools(name) DECLARE_FIELD(std::vector<bool>, name)
37
#define DECLARE_Int16s(name) DECLARE_FIELD(std::vector<int16_t>, name)
38
#define DECLARE_Int32s(name) DECLARE_FIELD(std::vector<int32_t>, name)
39
#define DECLARE_Int64s(name) DECLARE_FIELD(std::vector<int64_t>, name)
40
#define DECLARE_Doubles(name) DECLARE_FIELD(std::vector<double>, name)
41
#define DECLARE_Strings(name) DECLARE_FIELD(std::vector<std::string>, name)
42
#define DECLARE_mBool(name) DECLARE_FIELD(bool, name)
43
#define DECLARE_mInt16(name) DECLARE_FIELD(int16_t, name)
44
#define DECLARE_mInt32(name) DECLARE_FIELD(int32_t, name)
45
#define DECLARE_mInt64(name) DECLARE_FIELD(int64_t, name)
46
#define DECLARE_mDouble(name) DECLARE_FIELD(double, name)
47
#define DECLARE_mString(name) DECLARE_FIELD(std::string, name)
48
49
#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE)                      \
50
21
    FIELD_TYPE FIELD_NAME;                                                                   \
51
21
    static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &(FIELD_NAME), FIELD_DEFAULT, \
52
21
                                     VALMUTABLE);
53
54
#define DEFINE_VALIDATOR(FIELD_NAME, VALIDATOR)              \
55
    static auto validator_##FIELD_NAME = VALIDATOR;          \
56
    static RegisterConfValidator reg_validator_##FIELD_NAME( \
57
            #FIELD_NAME, []() -> bool { return validator_##FIELD_NAME(FIELD_NAME); });
58
59
1
#define DEFINE_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false)
60
1
#define DEFINE_Bools(name, defaultstr) DEFINE_FIELD(std::vector<bool>, name, defaultstr, false)
61
1
#define DEFINE_Doubles(name, defaultstr) DEFINE_FIELD(std::vector<double>, name, defaultstr, false)
62
1
#define DEFINE_Int16s(name, defaultstr) DEFINE_FIELD(std::vector<int16_t>, name, defaultstr, false)
63
1
#define DEFINE_Int32s(name, defaultstr) DEFINE_FIELD(std::vector<int32_t>, name, defaultstr, false)
64
1
#define DEFINE_Int64s(name, defaultstr) DEFINE_FIELD(std::vector<int64_t>, name, defaultstr, false)
65
3
#define DEFINE_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false)
66
1
#define DEFINE_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false)
67
1
#define DEFINE_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false)
68
1
#define DEFINE_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false)
69
2
#define DEFINE_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false)
70
#define DEFINE_Strings(name, defaultstr) \
71
1
    DEFINE_FIELD(std::vector<std::string>, name, defaultstr, false)
72
1
#define DEFINE_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true)
73
1
#define DEFINE_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true)
74
1
#define DEFINE_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true)
75
1
#define DEFINE_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true)
76
1
#define DEFINE_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true)
77
1
#define DEFINE_mString(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, true)
78
#define DEFINE_Validator(name, validator) DEFINE_VALIDATOR(name, validator)
79
80
namespace doris {
81
class Status;
82
83
// If you want to modify the value of config, please go to common/config.cpp to modify.
84
namespace config {
85
// Dir of custom config file
86
DECLARE_String(custom_config_dir);
87
88
// Dir of jdbc drivers
89
DECLARE_String(jdbc_drivers_dir);
90
91
// cluster id
92
DECLARE_Int32(cluster_id);
93
// port on which BackendService is exported
94
DECLARE_Int32(be_port);
95
96
// port for brpc
97
DECLARE_Int32(brpc_port);
98
99
// port for arrow flight sql
100
// Default -1, do not start arrow flight sql server.
101
DECLARE_Int32(arrow_flight_sql_port);
102
103
// If the external client cannot directly access priority_networks, set public_host to be accessible
104
// to external client.
105
// There are usually two usage scenarios:
106
// 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network.
107
// However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be
108
// randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx.
109
// 2. if priority_networks is an internal network IP, and BE node has its own independent external IP,
110
// but Doris currently does not support modifying priority_networks, setting public_host to the real external IP.
111
DECLARE_mString(public_host);
112
113
// If the BE node is connected to the external network through a reverse proxy like Nginx
114
// and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy
115
// `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example:
116
// upstream arrowflight {
117
//    server 10.16.10.8:8069;
118
//    server 10.16.10.8:8068;
119
//}
120
// server {
121
//    listen 8167 http2;
122
//    listen [::]:8167 http2;
123
//    server_name doris.arrowflight.com;
124
// }
125
DECLARE_Int32(arrow_flight_sql_proxy_port);
126
127
// the number of bthreads for brpc, the default value is set to -1,
128
// which means the number of bthreads is #cpu-cores
129
DECLARE_Int32(brpc_num_threads);
130
DECLARE_Int32(brpc_idle_timeout_sec);
131
132
// Declare a selection strategy for those servers have many ips.
133
// Note that there should at most one ip match this list.
134
// This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
135
// If no ip match this rule, will choose one randomly.
136
DECLARE_String(priority_networks);
137
138
// performance moderate or compact, only tcmalloc compile
139
DECLARE_String(memory_mode);
140
141
// if true, process memory limit and memory usage based on cgroup memory info.
142
DECLARE_mBool(enable_use_cgroup_memory_info);
143
144
// process memory limit specified as number of bytes
145
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
146
// or percentage of the physical memory ('<int>%').
147
// defaults to bytes if no unit is given"
148
// must larger than 0. and if larger than physical memory size,
149
// it will be set to physical memory size.
150
DECLARE_String(mem_limit);
151
152
// Soft memory limit as a fraction of hard memory limit.
153
DECLARE_Double(soft_mem_limit_frac);
154
155
// Cache capacity reduce mem limit as a fraction of soft mem limit.
156
DECLARE_mDouble(cache_capacity_reduce_mem_limit_frac);
157
158
// Schema change memory limit as a fraction of soft memory limit.
159
DECLARE_Double(schema_change_mem_limit_frac);
160
161
// Many modern allocators (for example) do not do a mremap for
162
// realloc, even in case of large enough chunks of memory. Although this allows
163
// you to increase performance and reduce memory consumption during realloc.
164
// To fix this, we do mremap manually if the chunk of memory is large enough.
165
//
166
// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
167
// space is very slow, especially in the case of a large number of threads. We
168
// expect that the set of operations mmap/something to do/mremap can only be
169
// performed about 1000 times per second.
170
//
171
// P.S. This is also required, because tcmalloc can not allocate a chunk of
172
// memory greater than 16 GB.
173
DECLARE_mInt64(mmap_threshold); // bytes
174
175
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
176
// Increase can reduce the number of hash table resize, but may waste more memory.
177
DECLARE_mInt32(hash_table_double_grow_degree);
178
179
// The max fill rate for hash table
180
DECLARE_mInt32(max_fill_rate);
181
182
DECLARE_mInt32(double_resize_threshold);
183
184
// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default -1.
185
// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%), which is 3.2G on a 64G machine.
186
// Turn up max. more memory buffers will be reserved for Memory GC.
187
// Turn down max. will use as much memory as possible.
188
// note that: `max_` prefix should be removed, but keep it for compatibility.
189
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
190
191
// reserve a small amount of memory so we do not trigger MinorGC
192
DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
193
194
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
195
DECLARE_mString(process_minor_gc_size);
196
DECLARE_mString(process_full_gc_size);
197
198
// If true, when the process does not exceed the soft mem limit, the query memory will not be limited;
199
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
200
// used memory and the exec_mem_limit will be canceled.
201
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
202
DECLARE_mBool(enable_query_memory_overcommit);
203
204
// gc will release cache, cancel task, and task will wait for gc to release memory,
205
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
206
DECLARE_mBool(disable_memory_gc);
207
208
// if false, turn off all stacktrace
209
DECLARE_mBool(enable_stacktrace);
210
211
// when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G,
212
// if alloc successful, will print a warning with stacktrace, but not prevent memory alloc.
213
// if alloc failed using Doris Allocator, will print stacktrace in error log.
214
// if is -1, disable print stacktrace when alloc large memory.
215
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
216
217
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled.
218
// if you need a core dump to analyze large memory allocation,
219
// modify this parameter to crash when large memory allocation occur will help
220
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
221
222
// The actual meaning of this parameter is `debug_memory`.
223
// 1. crash in memory tracker inaccurate, if memory tracker value is inaccurate, BE will crash.
224
//    usually used in test environments, default value is false.
225
// 2. print more memory logs.
226
DECLARE_mBool(crash_in_memory_tracker_inaccurate);
227
228
// default is true. if any memory tracking in Orphan mem tracker will report error.
229
// !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time.
230
// allocator free memory not need to check, because when the thread memory tracker label is Orphan,
231
// use the tracker saved in Allocator.
232
DECLARE_mBool(enable_memory_orphan_check);
233
234
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
235
DECLARE_mInt32(thread_wait_gc_max_milliseconds);
236
237
// reach mem limit, don't serialize in batch
238
DECLARE_mInt64(pre_serialize_keys_limit_bytes);
239
240
// the port heartbeat service used
241
DECLARE_Int32(heartbeat_service_port);
242
// the count of heart beat service
243
DECLARE_Int32(heartbeat_service_thread_count);
244
// the count of thread to create table
245
DECLARE_Int32(create_tablet_worker_count);
246
// the count of thread to drop table
247
DECLARE_Int32(drop_tablet_worker_count);
248
// the count of thread to batch load
249
DECLARE_Int32(push_worker_count_normal_priority);
250
// the count of thread to high priority batch load
251
DECLARE_Int32(push_worker_count_high_priority);
252
// the count of thread to publish version
253
DECLARE_Int32(publish_version_worker_count);
254
// the count of tablet thread to publish version
255
DECLARE_Int32(tablet_publish_txn_max_thread);
256
// the timeout of EnginPublishVersionTask
257
DECLARE_Int32(publish_version_task_timeout_s);
258
// the count of thread to calc delete bitmap
259
DECLARE_Int32(calc_delete_bitmap_max_thread);
260
// the count of thread to calc delete bitmap worker, only used for cloud
261
DECLARE_Int32(calc_delete_bitmap_worker_count);
262
// the count of thread to calc tablet delete bitmap task, only used for cloud
263
DECLARE_Int32(calc_tablet_delete_bitmap_task_max_thread);
264
// the count of thread to clear transaction task
265
DECLARE_Int32(clear_transaction_task_worker_count);
266
// the count of thread to delete
267
DECLARE_Int32(delete_worker_count);
268
// the count of thread to alter table
269
DECLARE_Int32(alter_tablet_worker_count);
270
// the count of thread to alter index
271
DECLARE_Int32(alter_index_worker_count);
272
// the count of thread to clone
273
DECLARE_Int32(clone_worker_count);
274
// the count of thread to clone
275
DECLARE_Int32(storage_medium_migrate_count);
276
// the count of thread to check consistency
277
DECLARE_Int32(check_consistency_worker_count);
278
// the count of thread to upload
279
DECLARE_Int32(upload_worker_count);
280
// the count of thread to download
281
DECLARE_Int32(download_worker_count);
282
// the count of thread to make snapshot
283
DECLARE_Int32(make_snapshot_worker_count);
284
// the count of thread to release snapshot
285
DECLARE_Int32(release_snapshot_worker_count);
286
// report random wait a little time to avoid FE receiving multiple be reports at the same time.
287
// do not set it to false for production environment
288
DECLARE_mBool(report_random_wait);
289
// the interval time(seconds) for agent report tasks signature to FE
290
DECLARE_mInt32(report_task_interval_seconds);
291
// the interval time(seconds) for agent report disk state to FE
292
DECLARE_mInt32(report_disk_state_interval_seconds);
293
// the interval time(seconds) for agent report olap table to FE
294
DECLARE_mInt32(report_tablet_interval_seconds);
295
// the max download speed(KB/s)
296
DECLARE_mInt32(max_download_speed_kbps);
297
// download low speed limit(KB/s)
298
DECLARE_mInt32(download_low_speed_limit_kbps);
299
// download low speed time(seconds)
300
DECLARE_mInt32(download_low_speed_time);
301
// whether to download small files in batch.
302
DECLARE_mBool(enable_batch_download);
303
// whether to check md5sum when download
304
DECLARE_mBool(enable_download_md5sum_check);
305
// download binlog meta timeout
306
DECLARE_mInt32(download_binlog_meta_timeout_ms);
307
308
// deprecated, use env var LOG_DIR in be.conf
309
DECLARE_String(sys_log_dir);
310
// for udf
311
DECLARE_String(user_function_dir);
312
// INFO, WARNING, ERROR, FATAL
313
DECLARE_String(sys_log_level);
314
// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
315
DECLARE_String(sys_log_roll_mode);
316
// log roll num
317
DECLARE_Int32(sys_log_roll_num);
318
// verbose log
319
DECLARE_Strings(sys_log_verbose_modules);
320
// verbose log level
321
DECLARE_Int32(sys_log_verbose_level);
322
// verbose log FLAGS_v
323
DECLARE_Int32(sys_log_verbose_flags_v);
324
// log buffer level
325
DECLARE_String(log_buffer_level);
326
// log enable custom date time format
327
DECLARE_Bool(sys_log_enable_custom_date_time_format);
328
// log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time)
329
DECLARE_String(sys_log_custom_date_time_format);
330
// log custom date time milliseconds format (fmt::format)
331
DECLARE_String(sys_log_custom_date_time_ms_format);
332
333
// number of threads available to serve backend execution requests
334
DECLARE_Int32(be_service_threads);
335
336
// interval between profile reports; in seconds
337
DECLARE_mInt32(pipeline_status_report_interval);
338
// number of scanner thread pool size for olap table
339
// and the min thread num of remote scanner thread pool
340
DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
341
DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num);
342
// number of batch size to fetch the remote split source
343
DECLARE_mInt32(remote_split_source_batch_size);
344
// max number of remote scanner thread pool size
345
// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
346
DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
347
// number of olap scanner thread pool queue size
348
DECLARE_Int32(doris_scanner_thread_pool_queue_size);
349
// default thrift client connect timeout(in seconds)
350
DECLARE_mInt32(thrift_connect_timeout_seconds);
351
DECLARE_mInt32(fetch_rpc_timeout_seconds);
352
// default thrift client retry interval (in milliseconds)
353
DECLARE_mInt64(thrift_client_retry_interval_ms);
354
// max message size of thrift request
355
// default: 100 * 1024 * 1024
356
DECLARE_mInt64(thrift_max_message_size);
357
// max bytes number for single scan range, used in segmentv2
358
DECLARE_mInt32(doris_scan_range_max_mb);
359
// single read execute fragment row number
360
DECLARE_mInt32(doris_scanner_row_num);
361
// single read execute fragment row bytes
362
DECLARE_mInt32(doris_scanner_row_bytes);
363
// single read execute fragment max run time millseconds
364
DECLARE_mInt32(doris_scanner_max_run_time_ms);
365
// (Advanced) Maximum size of per-query receive-side buffer
366
DECLARE_mInt32(exchg_node_buffer_size_bytes);
367
DECLARE_mInt32(exchg_buffer_queue_capacity_factor);
368
369
// memory_limitation_per_thread_for_schema_change_bytes unit bytes
370
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
371
372
// all cache prune interval, used by GC and periodic thread.
373
DECLARE_mInt32(cache_prune_interval_sec);
374
DECLARE_mInt32(cache_periodic_prune_stale_sweep_sec);
375
// the clean interval of tablet lookup cache
376
DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec);
377
DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec);
378
DECLARE_mInt32(disk_stat_monitor_interval);
379
DECLARE_mInt32(unused_rowset_monitor_interval);
380
DECLARE_mInt32(quering_rowsets_evict_interval);
381
DECLARE_String(storage_root_path);
382
DECLARE_mString(broken_storage_path);
383
384
// Config is used to check incompatible old format hdr_ format
385
// whether doris uses strict way. When config is true, process will log fatal
386
// and exit. When config is false, process will only log warning.
387
DECLARE_Bool(storage_strict_check_incompatible_old_format);
388
389
// BE process will exit if the percentage of error disk reach this value.
390
DECLARE_mInt32(max_percentage_of_error_disk);
391
DECLARE_mInt32(default_num_rows_per_column_file_block);
392
// pending data policy
393
DECLARE_mInt32(pending_data_expire_time_sec);
394
// inc_rowset snapshot rs sweep time interval
395
DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec);
396
// tablet stale rowset sweep by threshold size
397
DECLARE_Bool(tablet_rowset_stale_sweep_by_size);
398
DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size);
399
// garbage sweep policy
400
DECLARE_Int32(max_garbage_sweep_interval);
401
DECLARE_Int32(min_garbage_sweep_interval);
402
// garbage sweep every batch will sleep 1ms
403
DECLARE_mInt32(garbage_sweep_batch_size);
404
DECLARE_mInt32(snapshot_expire_time_sec);
405
// It is only a recommended value. When the disk space is insufficient,
406
// the file storage period under trash dose not have to comply with this parameter.
407
DECLARE_mInt32(trash_file_expire_time_sec);
408
// minimum file descriptor number
409
// modify them upon necessity
410
DECLARE_Int32(min_file_descriptor_number);
411
DECLARE_mBool(disable_segment_cache);
412
DECLARE_String(row_cache_mem_limit);
413
414
// Cache for storage page size
415
DECLARE_String(storage_page_cache_limit);
416
// Shard size for page cache, the value must be power of two.
417
// It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions.
418
DECLARE_Int32(storage_page_cache_shard_size);
419
// Percentage for index page cache
420
// all storage page cache will be divided into data_page_cache and index_page_cache
421
DECLARE_Int32(index_page_cache_percentage);
422
// whether to disable page cache feature in storage
423
// TODO delete it. Divided into Data page, Index page, pk index page
424
DECLARE_Bool(disable_storage_page_cache);
425
// whether to disable row cache feature in storage
426
DECLARE_mBool(disable_storage_row_cache);
427
// whether to disable pk page cache feature in storage
428
DECLARE_Bool(disable_pk_storage_page_cache);
429
430
// Cache for mow primary key storage page size, it's seperated from
431
// storage_page_cache_limit
432
DECLARE_String(pk_storage_page_cache_limit);
433
// data page size for primary key index
434
DECLARE_Int32(primary_key_data_page_size);
435
436
// inc_rowset snapshot rs sweep time interval
437
DECLARE_mInt32(data_page_cache_stale_sweep_time_sec);
438
DECLARE_mInt32(index_page_cache_stale_sweep_time_sec);
439
// great impact on the performance of MOW, so it can be longer.
440
DECLARE_mInt32(pk_index_page_cache_stale_sweep_time_sec);
441
442
DECLARE_Bool(enable_low_cardinality_optimize);
443
DECLARE_Bool(enable_low_cardinality_cache_code);
444
445
// be policy
446
// whether check compaction checksum
447
DECLARE_mBool(enable_compaction_checksum);
448
// whether disable automatic compaction task
449
DECLARE_mBool(disable_auto_compaction);
450
// whether enable vertical compaction
451
DECLARE_mBool(enable_vertical_compaction);
452
// whether enable ordered data compaction
453
DECLARE_mBool(enable_ordered_data_compaction);
454
// In vertical compaction, column number for every group
455
DECLARE_mInt32(vertical_compaction_num_columns_per_group);
456
// In vertical compaction, max memory usage for row_source_buffer
457
DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
458
// In vertical compaction, max dest segment file size
459
DECLARE_mInt64(vertical_compaction_max_segment_size);
460
461
// If enabled, segments will be flushed column by column
462
DECLARE_mBool(enable_vertical_segment_writer);
463
464
// In ordered data compaction, min segment size for input rowset
465
DECLARE_mInt32(ordered_data_compaction_min_segment_size);
466
467
// This config can be set to limit thread number in compaction thread pool.
468
DECLARE_mInt32(max_base_compaction_threads);
469
DECLARE_mInt32(max_cumu_compaction_threads);
470
DECLARE_mInt32(max_single_replica_compaction_threads);
471
472
DECLARE_Bool(enable_base_compaction_idle_sched);
473
DECLARE_mInt64(base_compaction_min_rowset_num);
474
DECLARE_mInt64(base_compaction_max_compaction_score);
475
DECLARE_mDouble(base_compaction_min_data_ratio);
476
DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes);
477
478
DECLARE_Bool(enable_skip_tablet_compaction);
479
DECLARE_mInt32(skip_tablet_compaction_second);
480
// output rowset of cumulative compaction total disk size exceed this config size,
481
// this rowset will be given to base compaction, unit is m byte.
482
DECLARE_mInt64(compaction_promotion_size_mbytes);
483
484
// output rowset of cumulative compaction total disk size exceed this config ratio of
485
// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
486
// 0 and 1.
487
DECLARE_mDouble(compaction_promotion_ratio);
488
489
// the smallest size of rowset promotion. When the rowset is less than this config, this
490
// rowset will be not given to base compaction. The unit is m byte.
491
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
492
493
// When output rowset of cumulative compaction total version count (end_version - start_version)
494
// exceed this config count, the rowset will be moved to base compaction
495
// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
496
// related cost on delete bitmap more effectively.
497
DECLARE_mInt64(compaction_promotion_version_count);
498
499
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
500
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
501
DECLARE_mInt64(compaction_min_size_mbytes);
502
503
// cumulative compaction policy: min and max delta file's number
504
DECLARE_mInt64(cumulative_compaction_min_deltas);
505
DECLARE_mInt64(cumulative_compaction_max_deltas);
506
DECLARE_mInt32(cumulative_compaction_max_deltas_factor);
507
508
// This config can be set to limit thread number in  multiget thread pool.
509
DECLARE_mInt32(multi_get_max_threads);
510
511
// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
512
DECLARE_mInt64(total_permits_for_compaction_score);
513
514
// sleep interval in ms after generated compaction tasks
515
DECLARE_mInt32(generate_compaction_tasks_interval_ms);
516
// sleep interval in second after update replica infos
517
DECLARE_mInt32(update_replica_infos_interval_seconds);
518
519
// Compaction task number per disk.
520
// Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each.
521
DECLARE_mInt32(compaction_task_num_per_disk);
522
// compaction thread num for fast disk(typically .SSD), must be greater than 2.
523
DECLARE_mInt32(compaction_task_num_per_fast_disk);
524
525
// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
526
DECLARE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round);
527
528
// Minimum number of threads required in the thread pool to activate the large cumu compaction delay strategy.
529
// The delay strategy is only applied when the thread pool has at least this many threads.
530
DECLARE_mInt32(large_cumu_compaction_task_min_thread_num);
531
// Maximum size threshold (in bytes) for input rowsets. Compaction tasks with input size
532
// exceeding this threshold will be delayed when thread pool is near capacity. Default 100MB.
533
DECLARE_mInt32(large_cumu_compaction_task_bytes_threshold);
534
// Maximum row count threshold for compaction input. Compaction tasks with row count
535
// exceeding this threshold will be delayed when thread pool is near capacity. Default 1 million.
536
DECLARE_mInt32(large_cumu_compaction_task_row_num_threshold);
537
538
// Not compact the invisible versions, but with some limitations:
539
// if not timeout, keep no more than compaction_keep_invisible_version_max_count versions;
540
// if timeout, keep no more than compaction_keep_invisible_version_min_count versions.
541
DECLARE_mInt32(compaction_keep_invisible_version_timeout_sec);
542
DECLARE_mInt32(compaction_keep_invisible_version_min_count);
543
DECLARE_mInt32(compaction_keep_invisible_version_max_count);
544
545
// Threshold to logging compaction trace, in seconds.
546
DECLARE_mInt32(base_compaction_trace_threshold);
547
DECLARE_mInt32(cumulative_compaction_trace_threshold);
548
DECLARE_mBool(disable_compaction_trace_log);
549
550
// Interval to picking rowset to compact, in seconds
551
DECLARE_mInt64(pick_rowset_to_compact_interval_sec);
552
553
// Compaction priority schedule
554
DECLARE_mBool(enable_compaction_priority_scheduling);
555
DECLARE_mInt32(low_priority_compaction_task_num_per_disk);
556
DECLARE_mInt32(low_priority_compaction_score_threshold);
557
558
// Thread count to do tablet meta checkpoint, -1 means use the data directories count.
559
DECLARE_Int32(max_meta_checkpoint_threads);
560
561
// Threshold to logging agent task trace, in seconds.
562
DECLARE_mInt32(agent_task_trace_threshold_sec);
563
564
// This config can be set to limit thread number in tablet migration thread pool.
565
DECLARE_Int32(min_tablet_migration_threads);
566
DECLARE_Int32(max_tablet_migration_threads);
567
568
DECLARE_mInt32(finished_migration_tasks_size);
569
// If size less than this, the remaining rowsets will be force to complete
570
DECLARE_mInt32(migration_remaining_size_threshold_mb);
571
// If the task runs longer than this time, the task will be terminated, in seconds.
572
// timeout = std::max(migration_task_timeout_secs,  tablet size / 1MB/s)
573
DECLARE_mInt32(migration_task_timeout_secs);
574
// timeout for try_lock migration lock
575
DECLARE_Int64(migration_lock_timeout_ms);
576
577
// Port to start debug webserver on
578
DECLARE_Int32(webserver_port);
579
// Https enable flag
580
DECLARE_Bool(enable_https);
581
// Path of certificate
582
DECLARE_String(ssl_certificate_path);
583
// Path of private key
584
DECLARE_String(ssl_private_key_path);
585
// Whether to check authorization
586
DECLARE_Bool(enable_all_http_auth);
587
// Number of webserver workers
588
DECLARE_Int32(webserver_num_workers);
589
590
DECLARE_Bool(enable_single_replica_load);
591
// Number of download workers for single replica load
592
DECLARE_Int32(single_replica_load_download_num_workers);
593
594
// Used for mini Load. mini load data file will be removed after this time.
595
DECLARE_Int64(load_data_reserve_hours);
596
// log error log will be removed after this time
597
DECLARE_mInt64(load_error_log_reserve_hours);
598
// error log size limit, default 200MB
599
DECLARE_mInt64(load_error_log_limit_bytes);
600
601
// be brpc interface is classified into two categories: light and heavy
602
// each category has diffrent thread number
603
// threads to handle heavy api interface, such as transmit_block etc
604
DECLARE_Int32(brpc_heavy_work_pool_threads);
605
// threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start
606
DECLARE_Int32(brpc_light_work_pool_threads);
607
DECLARE_Int32(brpc_heavy_work_pool_max_queue_size);
608
DECLARE_Int32(brpc_light_work_pool_max_queue_size);
609
DECLARE_mBool(enable_bthread_transmit_block);
610
DECLARE_Int32(brpc_arrow_flight_work_pool_threads);
611
DECLARE_Int32(brpc_arrow_flight_work_pool_max_queue_size);
612
613
// The maximum amount of data that can be processed by a stream load
614
DECLARE_mInt64(streaming_load_max_mb);
615
// Some data formats, such as JSON, cannot be streamed.
616
// Therefore, it is necessary to limit the maximum number of
617
// such data when using stream load to prevent excessive memory consumption.
618
DECLARE_mInt64(streaming_load_json_max_mb);
619
// the alive time of a TabletsChannel.
620
// If the channel does not receive any data till this time,
621
// the channel will be removed.
622
DECLARE_mInt32(streaming_load_rpc_max_alive_time_sec);
623
// the timeout of a rpc to open the tablet writer in remote BE.
624
// short operation time, can set a short timeout
625
DECLARE_Int32(tablet_writer_open_rpc_timeout_sec);
626
// You can ignore brpc error '[E1011]The server is overcrowded' when writing data.
627
DECLARE_mBool(tablet_writer_ignore_eovercrowded);
628
DECLARE_mInt32(slave_replica_writer_rpc_timeout_sec);
629
// Whether to enable stream load record function, the default is false.
630
// False: disable stream load record
631
DECLARE_mBool(enable_stream_load_record);
632
// batch size of stream load record reported to FE
633
DECLARE_mInt32(stream_load_record_batch_size);
634
// expire time of stream load record in rocksdb.
635
DECLARE_Int32(stream_load_record_expire_time_secs);
636
// time interval to clean expired stream load records
637
DECLARE_mInt64(clean_stream_load_record_interval_secs);
638
// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
639
DECLARE_mBool(enable_stream_load_commit_txn_on_be);
640
// The buffer size to store stream table function schema info
641
DECLARE_Int64(stream_tvf_buffer_size);
642
643
// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
644
// You may need to lower the speed when the sink receiver bes are too busy.
645
DECLARE_mInt32(olap_table_sink_send_interval_microseconds);
646
// For auto partition, the send interval will multiply the factor
647
DECLARE_mDouble(olap_table_sink_send_interval_auto_partition_factor);
648
649
// Fragment thread pool
650
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_min);
651
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_max);
652
DECLARE_Int32(fragment_mgr_asynic_work_pool_queue_size);
653
654
// Control the number of disks on the machine.  If 0, this comes from the system settings.
655
DECLARE_Int32(num_disks);
656
// The read size is the size of the reads sent to os.
657
// There is a trade off of latency and throughout, trying to keep disks busy but
658
// not introduce seeks.  The literature seems to agree that with 8 MB reads, random
659
// io and sequential io perform similarly.
660
DECLARE_Int32(read_size);       // 8 * 1024 * 1024, Read Size (in bytes)
661
DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
662
663
// for pprof
664
DECLARE_String(pprof_profile_dir);
665
// for jeprofile in jemalloc
666
DECLARE_mString(jeprofile_dir);
667
// Purge all unused dirty pages for all arenas.
668
DECLARE_mBool(enable_je_purge_dirty_pages);
669
// Jemalloc `arenas.dirty_decay_ms`, equal to `dirty_decay_ms` in JEMALLOC_CONF in be.conf.
670
DECLARE_mInt32(je_dirty_decay_ms);
671
672
// to forward compatibility, will be removed later
673
DECLARE_mBool(enable_token_check);
674
675
// to open/close system metrics
676
DECLARE_Bool(enable_system_metrics);
677
678
// Number of cores Doris will used, this will effect only when it's greater than 0.
679
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
680
DECLARE_Int32(num_cores);
681
682
// When BE start, If there is a broken disk, BE process will exit by default.
683
// Otherwise, we will ignore the broken disk,
684
DECLARE_Bool(ignore_broken_disk);
685
686
// Sleep time in milliseconds between memory maintenance iterations
687
DECLARE_mInt32(memory_maintenance_sleep_time_ms);
688
689
// After full gc, no longer full gc and minor gc during sleep.
690
// After minor gc, no minor gc during sleep, but full gc is possible.
691
DECLARE_mInt32(memory_gc_sleep_time_ms);
692
693
// max write buffer size before flush, default 200MB
694
DECLARE_mInt64(write_buffer_size);
695
// max buffer size used in memtable for the aggregated table, default 400MB
696
DECLARE_mInt64(write_buffer_size_for_agg);
697
// max parallel flush task per memtable writer
698
DECLARE_mInt32(memtable_flush_running_count_limit);
699
700
DECLARE_Int32(load_process_max_memory_limit_percent); // 50%
701
702
// If the memory consumption of load jobs exceed load_process_max_memory_limit,
703
// all load jobs will hang there to wait for memtable flush. We should have a
704
// soft limit which can trigger the memtable flush for the load channel who
705
// consumes lagest memory size before we reach the hard limit. The soft limit
706
// might avoid all load jobs hang at the same time.
707
DECLARE_Int32(load_process_soft_mem_limit_percent);
708
709
// If load memory consumption is within load_process_safe_mem_permit_percent,
710
// memtable memory limiter will do nothing.
711
DECLARE_Int32(load_process_safe_mem_permit_percent);
712
713
// If there are a lot of memtable memory, then wait them flush finished.
714
DECLARE_mDouble(load_max_wg_active_memtable_percent);
715
716
// result buffer cancelled time (unit: second)
717
DECLARE_mInt32(result_buffer_cancelled_interval_time);
718
719
// arrow flight result sink buffer rows size, default 4096 * 8
720
DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows);
721
// The timeout for ADBC Client to wait for data using arrow flight reader.
722
// If the query is very complex and no result is generated after this time, consider increasing this timeout.
723
DECLARE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms);
724
725
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
726
DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency);
727
728
// sync tablet_meta when modifying meta
729
DECLARE_mBool(sync_tablet_meta);
730
731
// sync a file writer when it is closed
732
DECLARE_mBool(sync_file_on_close);
733
734
// default thrift rpc timeout ms
735
DECLARE_mInt32(thrift_rpc_timeout_ms);
736
737
// txn commit rpc timeout
738
DECLARE_mInt32(txn_commit_rpc_timeout_ms);
739
740
// If set to true, metric calculator will run
741
DECLARE_Bool(enable_metric_calculator);
742
743
// max consumer num in one data consumer group, for routine load
744
DECLARE_mInt32(max_consumer_num_per_group);
745
746
// the max size of thread pool for routine load task.
747
// this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5)
748
DECLARE_Int32(max_routine_load_thread_pool_size);
749
750
// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
751
// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
752
DECLARE_mInt32(max_memory_sink_batch_count);
753
754
// This configuration is used for the context gc thread schedule period
755
// note: unit is minute, default is 5min
756
DECLARE_mInt32(scan_context_gc_interval_min);
757
758
// es scroll keep-alive
759
DECLARE_String(es_scroll_keepalive);
760
761
// HTTP connection timeout for es
762
DECLARE_mInt32(es_http_timeout_ms);
763
764
// the max client cache number per each host
765
// There are variety of client cache in BE, but currently we use the
766
// same cache size configuration.
767
// TODO(cmy): use different config to set different client cache if necessary.
768
DECLARE_Int32(max_client_cache_size_per_host);
769
770
DECLARE_Int32(max_master_fe_client_cache_size);
771
772
// Dir to save files downloaded by SmallFileMgr
773
DECLARE_String(small_file_dir);
774
// path gc
775
DECLARE_Bool(path_gc_check);
776
DECLARE_mInt32(path_gc_check_interval_second);
777
DECLARE_mInt32(path_gc_check_step);
778
DECLARE_mInt32(path_gc_check_step_interval_ms);
779
780
// The following 2 configs limit the max usage of disk capacity of a data dir.
781
// If both of these 2 threshold reached, no more data can be writen into that data dir.
782
// The percent of max used capacity of a data dir
783
DECLARE_mInt32(storage_flood_stage_usage_percent); // 90%
784
// The min bytes that should be left of a data dir
785
DECLARE_mInt64(storage_flood_stage_left_capacity_bytes); // 1GB
786
// number of thread for flushing memtable per store
787
DECLARE_Int32(flush_thread_num_per_store);
788
// number of thread for flushing memtable per store, for high priority load task
789
DECLARE_Int32(high_priority_flush_thread_num_per_store);
790
// number of threads = min(flush_thread_num_per_store * num_store,
791
//                         max_flush_thread_num_per_cpu * num_cpu)
792
DECLARE_Int32(max_flush_thread_num_per_cpu);
793
794
// workload group flush pool params
795
DECLARE_mInt32(wg_flush_thread_num_per_store);
796
DECLARE_mInt32(wg_flush_thread_num_per_cpu);
797
798
// config for tablet meta checkpoint
799
DECLARE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num);
800
DECLARE_mInt32(tablet_meta_checkpoint_min_interval_secs);
801
DECLARE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs);
802
803
// config for default rowset type
804
// Valid configs: ALPHA, BETA
805
DECLARE_String(default_rowset_type);
806
807
// Maximum size of a single message body in all protocols
808
DECLARE_Int64(brpc_max_body_size);
809
// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
810
// Default, if the physical memory is less than or equal to 64G, the value is 1G
811
//          if the physical memory is greater than 64G, the value is physical memory * mem_limit(0.8) / 1024 * 20
812
DECLARE_Int64(brpc_socket_max_unwritten_bytes);
813
// TODO(zxy): expect to be true in v1.3
814
// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
815
// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
816
// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
817
DECLARE_mBool(transfer_large_data_by_brpc);
818
819
// max number of txns for every txn_partition_map in txn manager
820
// this is a self protection to avoid too many txns saving in manager
821
DECLARE_mInt64(max_runnings_transactions_per_txn_map);
822
823
// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
824
// this is a an enhancement for better performance to manage tablet
825
DECLARE_Int32(tablet_map_shard_size);
826
827
// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
828
// this is a an enhancement for better performance to manage txn
829
DECLARE_Int32(txn_map_shard_size);
830
831
// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
832
// this is a an enhancement for better performance to commit and publish txn
833
DECLARE_Int32(txn_shard_size);
834
835
// Whether to continue to start be when load tablet from header failed.
836
DECLARE_Bool(ignore_load_tablet_failure);
837
838
// Whether to continue to start be when load tablet from header failed.
839
DECLARE_mBool(ignore_rowset_stale_unconsistent_delete);
840
841
// Set max cache's size of query results, the unit is M byte
842
DECLARE_Int32(query_cache_max_size_mb);
843
844
// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
845
DECLARE_Int32(query_cache_elasticity_size_mb);
846
847
// Maximum number of cache partitions corresponding to a SQL
848
DECLARE_Int32(query_cache_max_partition_count);
849
850
// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
851
// the load process will reject new incoming load job of this tablet.
852
// This is to avoid too many version num.
853
DECLARE_mInt32(max_tablet_version_num);
854
855
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
856
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
857
DECLARE_String(thrift_server_type_of_fe);
858
859
// disable zone map index when page row is too few
860
DECLARE_mInt32(zone_map_row_num_threshold);
861
862
// aws sdk log level
863
//    Off = 0,
864
//    Fatal = 1,
865
//    Error = 2,
866
//    Warn = 3,
867
//    Info = 4,
868
//    Debug = 5,
869
//    Trace = 6
870
DECLARE_Int32(aws_log_level);
871
872
// the buffer size when read data from remote storage like s3
873
DECLARE_mInt32(remote_storage_read_buffer_mb);
874
875
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
876
// smaller than this value will continue to accumulate. specified as number of bytes.
877
// Decreasing this value will increase the frequency of consume/release.
878
// Increasing this value will cause MemTracker statistics to be inaccurate.
879
DECLARE_mInt32(mem_tracker_consume_min_size_bytes);
880
881
// The version information of the tablet will be stored in the memory
882
// in an adjacency graph data structure.
883
// And as the new version is written and the old version is deleted,
884
// the data structure will begin to have empty vertex with no edge associations(orphan vertex).
885
// This config is used to control that when the proportion of orphan vertex is greater than the threshold,
886
// the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely.
887
// This config usually only needs to be modified during testing.
888
// In most cases, it does not need to be modified.
889
DECLARE_mDouble(tablet_version_graph_orphan_vertex_ratio);
890
891
// share delta writers when memtable_on_sink_node = true
892
DECLARE_Bool(share_delta_writers);
893
// timeout for open load stream rpc in ms
894
DECLARE_Int64(open_load_stream_timeout_ms);
895
// enable write background when using brpc stream
896
DECLARE_mBool(enable_brpc_stream_write_background);
897
898
// brpc streaming max_buf_size in bytes
899
DECLARE_Int64(load_stream_max_buf_size);
900
// brpc streaming messages_in_batch
901
DECLARE_Int32(load_stream_messages_in_batch);
902
// brpc streaming StreamWait seconds on EAGAIN
903
DECLARE_Int32(load_stream_eagain_wait_seconds);
904
// max tasks per flush token in load stream
905
DECLARE_Int32(load_stream_flush_token_max_tasks);
906
// max wait flush token time in load stream
907
DECLARE_Int32(load_stream_max_wait_flush_token_time_ms);
908
// number of send batch thread pool size
909
DECLARE_Int32(send_batch_thread_pool_thread_num);
910
// number of send batch thread pool queue size
911
DECLARE_Int32(send_batch_thread_pool_queue_size);
912
913
// Limit the number of segment of a newly created rowset.
914
// The newly created rowset may to be compacted after loading,
915
// so if there are too many segment in a rowset, the compaction process
916
// will run out of memory.
917
// When doing compaction, each segment may take at least 1MB buffer.
918
DECLARE_mInt32(max_segment_num_per_rowset);
919
920
// Store segment without compression if a segment is smaller than
921
// segment_compression_threshold_kb.
922
DECLARE_mInt32(segment_compression_threshold_kb);
923
924
// The connection timeout when connecting to external table such as odbc table.
925
DECLARE_mInt32(external_table_connect_timeout_sec);
926
927
// Time to clean up useless JDBC connection pool cache
928
DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
929
930
// Global bitmap cache capacity for aggregation cache, size in bytes
931
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
932
DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
933
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
934
935
// A common object cache depends on an Sharded LRU Cache.
936
DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
937
938
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
939
// If the dependent kafka broker version older than 0.10.0.0,
940
// the value of kafka_api_version_request should be false, and the
941
// value set by the fallback version kafka_broker_version_fallback will be used,
942
// and the valid values are: 0.9.0.x, 0.8.x.y.
943
DECLARE_String(kafka_api_version_request);
944
DECLARE_String(kafka_broker_version_fallback);
945
DECLARE_mString(kafka_debug);
946
947
// The number of pool siz of routine load consumer.
948
// If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608
949
// Change this size to 0 to fix it temporarily.
950
DECLARE_mInt32(routine_load_consumer_pool_size);
951
952
// Used in single-stream-multi-table load. When receive a batch of messages from kafka,
953
// if the size of batch is more than this threshold, we will request plans for all related tables.
954
DECLARE_Int32(multi_table_batch_plan_threshold);
955
956
// Used in single-stream-multi-table load. When receiving a batch of messages from Kafka,
957
// if the size of the table wait for plan is more than this threshold, we will request plans for all related tables.
958
// The param is aimed to avoid requesting and executing too many plans at once.
959
// Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC
960
// and improve the real-time processing of data.
961
DECLARE_Int32(multi_table_max_wait_tables);
962
963
// When the timeout of a load task is less than this threshold,
964
// Doris treats it as a high priority task.
965
// high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic.
966
// this threshold is mainly used to identify routine load tasks and should not be modified if not necessary.
967
DECLARE_mInt32(load_task_high_priority_threshold_second);
968
969
// The min timeout of load rpc (add batch, close)
970
// Because a load rpc may be blocked for a while.
971
// Increase this config may avoid rpc timeout.
972
DECLARE_mInt32(min_load_rpc_timeout_ms);
973
974
// use which protocol to access function service, candicate is baidu_std/h2:grpc
975
DECLARE_String(function_service_protocol);
976
977
// use which load balancer to select server to connect
978
DECLARE_String(rpc_load_balancer);
979
980
// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
981
// so we set a soft limit, default is 1MB
982
DECLARE_mInt32(string_type_length_soft_limit_bytes);
983
984
// Threshold fo reading a small file into memory
985
DECLARE_mInt32(in_memory_file_size);
986
987
// Max size of parquet page header in bytes
988
DECLARE_mInt32(parquet_header_max_size_mb);
989
// Max buffer size for parquet row group
990
DECLARE_mInt32(parquet_rowgroup_max_buffer_mb);
991
// Max buffer size for parquet chunk column
992
DECLARE_mInt32(parquet_column_max_buffer_mb);
993
// Merge small IO, the max amplified read ratio
994
DECLARE_mDouble(max_amplified_read_ratio);
995
// Equivalent min size of each IO that can reach the maximum storage speed limit
996
// 1MB for oss, 8KB for hdfs
997
DECLARE_mInt32(merged_oss_min_io_size);
998
DECLARE_mInt32(merged_hdfs_min_io_size);
999
1000
// OrcReader
1001
DECLARE_mInt32(orc_natural_read_size_mb);
1002
DECLARE_mInt64(big_column_size_buffer);
1003
DECLARE_mInt64(small_column_size_buffer);
1004
1005
DECLARE_mInt32(runtime_filter_sampling_frequency);
1006
DECLARE_mInt32(execution_max_rpc_timeout_sec);
1007
DECLARE_mBool(execution_ignore_eovercrowded);
1008
1009
// cooldown task configs
1010
DECLARE_Int32(cooldown_thread_num);
1011
DECLARE_mInt64(generate_cooldown_task_interval_sec);
1012
DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h
1013
DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
1014
DECLARE_Int32(cold_data_compaction_thread_num);
1015
DECLARE_mInt32(cold_data_compaction_interval_sec);
1016
1017
DECLARE_Int32(min_s3_file_system_thread_num);
1018
DECLARE_Int32(max_s3_file_system_thread_num);
1019
1020
DECLARE_Bool(enable_time_lut);
1021
DECLARE_mBool(enable_simdjson_reader);
1022
1023
DECLARE_mBool(enable_query_like_bloom_filter);
1024
// number of s3 scanner thread pool size
1025
DECLARE_Int32(doris_remote_scanner_thread_pool_thread_num);
1026
// number of s3 scanner thread pool queue size
1027
DECLARE_Int32(doris_remote_scanner_thread_pool_queue_size);
1028
1029
// limit the queue of pending batches which will be sent by a single nodechannel
1030
DECLARE_mInt64(nodechannel_pending_queue_max_bytes);
1031
1032
// The batch size for sending data by brpc streaming client
1033
DECLARE_mInt64(brpc_streaming_client_batch_bytes);
1034
DECLARE_mInt64(block_cache_wait_timeout_ms);
1035
1036
DECLARE_Bool(enable_brpc_builtin_services);
1037
1038
DECLARE_Bool(enable_brpc_connection_check);
1039
1040
DECLARE_mInt64(brpc_connection_check_timeout_ms);
1041
1042
// Max waiting time to wait the "plan fragment start" rpc.
1043
// If timeout, the fragment will be cancelled.
1044
// This parameter is usually only used when the FE loses connection,
1045
// and the BE can automatically cancel the relevant fragment after the timeout,
1046
// so as to avoid occupying the execution thread for a long time.
1047
DECLARE_mInt32(max_fragment_start_wait_time_seconds);
1048
1049
DECLARE_Int32(fragment_mgr_cancel_worker_interval_seconds);
1050
1051
// Node role tag for backend. Mix role is the default role, and computation role have no
1052
// any tablet.
1053
DECLARE_String(be_node_role);
1054
1055
// Hide webserver page for safety.
1056
// Hide the be config page for webserver.
1057
DECLARE_Bool(hide_webserver_config_page);
1058
1059
DECLARE_Bool(enable_segcompaction);
1060
1061
// Max number of segments allowed in a single segcompaction task.
1062
DECLARE_mInt32(segcompaction_batch_size);
1063
1064
// Max row count allowed in a single source segment, bigger segments will be skipped.
1065
DECLARE_Int32(segcompaction_candidate_max_rows);
1066
1067
// Max file size allowed in a single source segment, bigger segments will be skipped.
1068
DECLARE_Int64(segcompaction_candidate_max_bytes);
1069
1070
// Max total row count allowed in a single segcompaction task.
1071
DECLARE_Int32(segcompaction_task_max_rows);
1072
1073
// Max total file size allowed in a single segcompaction task.
1074
DECLARE_Int64(segcompaction_task_max_bytes);
1075
1076
// Global segcompaction thread pool size.
1077
DECLARE_mInt32(segcompaction_num_threads);
1078
1079
// enable java udf and jdbc scannode
1080
DECLARE_Bool(enable_java_support);
1081
1082
// enable prefetch tablets before opening
1083
DECLARE_mBool(enable_prefetch_tablet);
1084
1085
// Set config randomly to check more issues in github workflow
1086
DECLARE_Bool(enable_fuzzy_mode);
1087
1088
DECLARE_Bool(enable_debug_points);
1089
1090
DECLARE_Int32(pipeline_executor_size);
1091
1092
// block file cache
1093
DECLARE_Bool(enable_file_cache);
1094
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
1095
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
1096
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}]
1097
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
1098
// Note1: storage is "disk" by default
1099
// Note2: when the storage is "memory", the path is ignored. So you can set xxx to anything you like
1100
// and doris will just reset the path to "memory" internally.
1101
// In a very wierd case when your storage is disk, and the directory, by accident, is named
1102
// "memory" for some reason, you should write the path as:
1103
//     {"path": "memory", "total_size":53687091200, "storage": "disk"}
1104
// or use the default storage value:
1105
//     {"path": "memory", "total_size":53687091200}
1106
// Both will use the directory "memory" on the disk instead of the real RAM.
1107
DECLARE_String(file_cache_path);
1108
DECLARE_Int64(file_cache_each_block_size);
1109
DECLARE_Bool(clear_file_cache);
1110
DECLARE_Bool(enable_file_cache_query_limit);
1111
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
1112
DECLARE_Int32(file_cache_exit_disk_resource_limit_mode_percent);
1113
DECLARE_mBool(enable_evict_file_cache_in_advance);
1114
DECLARE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent);
1115
DECLARE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent);
1116
DECLARE_mInt32(file_cache_evict_in_advance_interval_ms);
1117
DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes);
1118
DECLARE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold);
1119
DECLARE_mBool(enable_read_cache_file_directly);
1120
DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size);
1121
// If true, evict the ttl cache using LRU when full.
1122
// Otherwise, only expiration can evict ttl and new data won't add to cache when full.
1123
DECLARE_Bool(enable_ttl_cache_evict_using_lru);
1124
DECLARE_mBool(enbale_dump_error_file);
1125
// limit the max size of error log on disk
1126
DECLARE_mInt64(file_cache_error_log_limit_bytes);
1127
DECLARE_mInt64(cache_lock_wait_long_tail_threshold_us);
1128
DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
1129
// Base compaction may retrieve and produce some less frequently accessed data,
1130
// potentially affecting the file cache hit rate.
1131
// This configuration determines whether to retain the output within the file cache.
1132
// Make your choice based on the following considerations:
1133
// If your file cache is ample enough to accommodate all the data in your database,
1134
// enable this option; otherwise, it is recommended to leave it disabled.
1135
DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
1136
DECLARE_mInt64(file_cache_remove_block_qps_limit);
1137
DECLARE_mInt64(file_cache_background_gc_interval_ms);
1138
// inverted index searcher cache
1139
// cache entry stay time after lookup
1140
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
1141
// cache entry that have not been visited for a certain period of time can be cleaned up by GC thread
1142
DECLARE_mInt32(inverted_index_cache_stale_sweep_time_sec);
1143
// inverted index searcher cache size
1144
DECLARE_String(inverted_index_searcher_cache_limit);
1145
DECLARE_mBool(enable_write_index_searcher_cache);
1146
DECLARE_Bool(enable_inverted_index_cache_check_timestamp);
1147
DECLARE_mBool(enable_inverted_index_correct_term_write);
1148
DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50%
1149
DECLARE_Int32(inverted_index_query_cache_shards);
1150
1151
// inverted index match bitmap cache size
1152
DECLARE_String(inverted_index_query_cache_limit);
1153
1154
// inverted index
1155
DECLARE_mDouble(inverted_index_ram_buffer_size);
1156
DECLARE_mInt32(inverted_index_max_buffered_docs);
1157
// dict path for chinese analyzer
1158
DECLARE_String(inverted_index_dict_path);
1159
DECLARE_Int32(inverted_index_read_buffer_size);
1160
// tree depth for bkd index
1161
DECLARE_Int32(max_depth_in_bkd_tree);
1162
// index compaction
1163
DECLARE_mBool(inverted_index_compaction_enable);
1164
// Only for debug, do not use in production
1165
DECLARE_mBool(debug_inverted_index_compaction);
1166
// index by RAM directory
1167
DECLARE_mBool(inverted_index_ram_dir_enable);
1168
// wheather index by RAM directory when base compaction
1169
DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction);
1170
// use num_broadcast_buffer blocks as buffer to do broadcast
1171
DECLARE_Int32(num_broadcast_buffer);
1172
1173
// max depth of expression tree allowed.
1174
DECLARE_Int32(max_depth_of_expr_tree);
1175
1176
// Report a tablet as bad when io errors occurs more than this value.
1177
DECLARE_mInt64(max_tablet_io_errors);
1178
1179
// Report a tablet as bad when its path not found
1180
DECLARE_Int32(tablet_path_check_interval_seconds);
1181
DECLARE_mInt32(tablet_path_check_batch_size);
1182
1183
// it must be larger than or equal to 5MB
1184
DECLARE_mInt64(s3_write_buffer_size);
1185
// Log interval when doing s3 upload task
1186
DECLARE_mInt32(s3_file_writer_log_interval_second);
1187
// the max number of cached file handle for block segemnt
1188
DECLARE_mInt64(file_cache_max_file_reader_cache_size);
1189
DECLARE_mInt64(hdfs_write_batch_buffer_size_mb);
1190
//enable shrink memory
1191
DECLARE_mBool(enable_shrink_memory);
1192
// enable cache for high concurrent point query work load
1193
DECLARE_mInt32(schema_cache_capacity);
1194
DECLARE_mInt32(schema_cache_sweep_time_sec);
1195
1196
// max number of segment cache
1197
DECLARE_Int32(segment_cache_capacity);
1198
DECLARE_Int32(segment_cache_fd_percentage);
1199
DECLARE_Int32(segment_cache_memory_percentage);
1200
DECLARE_mInt32(estimated_mem_per_column_reader);
1201
1202
// enable binlog
1203
DECLARE_Bool(enable_feature_binlog);
1204
1205
// enable set in BitmapValue
1206
DECLARE_Bool(enable_set_in_bitmap_value);
1207
1208
// max number of hdfs file handle in cache
1209
DECLARE_Int64(max_hdfs_file_handle_cache_num);
1210
DECLARE_Int32(max_hdfs_file_handle_cache_time_sec);
1211
1212
// max number of meta info of external files, such as parquet footer
1213
DECLARE_Int64(max_external_file_meta_cache_num);
1214
// Apply delete pred in cumu compaction
1215
DECLARE_mBool(enable_delete_when_cumu_compaction);
1216
1217
// max_write_buffer_number for rocksdb
1218
DECLARE_Int32(rocksdb_max_write_buffer_number);
1219
1220
// Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false.
1221
DECLARE_mBool(allow_zero_date);
1222
// Allow invalid decimalv2 literal for compatible with old version. Recommend set it false strongly.
1223
DECLARE_mBool(allow_invalid_decimalv2_literal);
1224
// Allow to specify kerberos credentials cache path.
1225
DECLARE_mString(kerberos_ccache_path);
1226
// set krb5.conf path, use "/etc/krb5.conf" by default
1227
DECLARE_mString(kerberos_krb5_conf_path);
1228
// the interval for renew kerberos ticket cache
1229
DECLARE_mInt32(kerberos_refresh_interval_second);
1230
1231
// Values include `none`, `glog`, `boost`, `glibc`, `libunwind`
1232
DECLARE_mString(get_stack_trace_tool);
1233
DECLARE_mBool(enable_address_sanitizers_with_stack_trace);
1234
1235
// DISABLED: Don't resolve location info.
1236
// FAST: Perform CU lookup using .debug_aranges (might be incomplete).
1237
// FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure.
1238
// FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL.
1239
DECLARE_mString(dwarf_location_info_mode);
1240
1241
// the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
1242
DECLARE_mInt64(auto_inc_prefetch_size_ratio);
1243
1244
// the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer
1245
DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
1246
1247
// number of threads that fetch auto-inc ranges from FE
1248
DECLARE_mInt64(auto_inc_fetch_thread_num);
1249
// Max connection cache num for point lookup queries
1250
DECLARE_mInt64(lookup_connection_cache_capacity);
1251
1252
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
1253
DECLARE_mInt64(LZ4_HC_compression_level);
1254
// Threshold of a column as sparse column
1255
// Notice: TEST ONLY
1256
DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column);
1257
DECLARE_mBool(variant_use_cloud_schema_dict_cache);
1258
// Threshold to estimate a column is sparsed
1259
// Notice: TEST ONLY
1260
DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column);
1261
// Treat invalid json format str as string, instead of throwing exception if false
1262
DECLARE_mBool(variant_throw_exeception_on_invalid_json);
1263
1264
DECLARE_mBool(enable_merge_on_write_correctness_check);
1265
// USED FOR DEBUGING
1266
// core directly if the compaction found there's duplicate key on mow table
1267
DECLARE_mBool(enable_mow_compaction_correctness_check_core);
1268
// rowid conversion correctness check when compaction for mow table
1269
DECLARE_mBool(enable_rowid_conversion_correctness_check);
1270
// missing rows correctness check when compaction for mow table
1271
DECLARE_mBool(enable_missing_rows_correctness_check);
1272
// When the number of missing versions is more than this value, do not directly
1273
// retry the publish and handle it through async publish.
1274
DECLARE_mInt32(mow_publish_max_discontinuous_version_num);
1275
// When the size of primary keys in memory exceeds this value, finish current segment
1276
// and create a new segment, used in compaction.
1277
DECLARE_mInt64(mow_primary_key_index_max_size_in_memory);
1278
// When the version is not continuous for MOW table in publish phase and the gap between
1279
// current txn's publishing version and the max version of the tablet exceeds this value,
1280
// don't print warning log
1281
DECLARE_mInt32(publish_version_gap_logging_threshold);
1282
// get agg by cache for mow table
1283
DECLARE_mBool(enable_mow_get_agg_by_cache);
1284
// get agg correctness check for mow table
1285
DECLARE_mBool(enable_mow_get_agg_correctness_check_core);
1286
1287
// The secure path with user files, used in the `local` table function.
1288
DECLARE_mString(user_files_secure_path);
1289
1290
// If fe's frontend info has not been updated for more than fe_expire_duration_seconds, it will be regarded
1291
// as an abnormal fe, this will cause be to cancel this fe's related query.
1292
DECLARE_Int32(fe_expire_duration_seconds);
1293
1294
// If use stop_be.sh --grace, then BE has to wait all running queries to stop to avoiding running query failure
1295
// , but if the waiting time exceed the limit, then be will exit directly.
1296
// During this period, FE will not send any queries to BE and waiting for all running queries to stop.
1297
DECLARE_Int32(grace_shutdown_wait_seconds);
1298
1299
// BitmapValue serialize version.
1300
DECLARE_Int16(bitmap_serialize_version);
1301
1302
// group commit config
1303
DECLARE_String(group_commit_wal_path);
1304
DECLARE_Int32(group_commit_replay_wal_retry_num);
1305
DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds);
1306
DECLARE_Int32(group_commit_replay_wal_retry_interval_max_seconds);
1307
DECLARE_mInt32(group_commit_relay_wal_threads);
1308
// This config can be set to limit thread number in group commit request fragment thread pool.
1309
DECLARE_mInt32(group_commit_insert_threads);
1310
DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
1311
DECLARE_Bool(wait_internal_group_commit_finish);
1312
// Max size(bytes) of group commit queues, used for mem back pressure.
1313
DECLARE_mInt32(group_commit_queue_mem_limit);
1314
// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
1315
// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
1316
DECLARE_mString(group_commit_wal_max_disk_limit);
1317
DECLARE_Bool(group_commit_wait_replay_wal_finish);
1318
1319
// The configuration item is used to lower the priority of the scanner thread,
1320
// typically employed to ensure CPU scheduling for write operations.
1321
// Default is 0, which is default value of thread nice value, increase this value
1322
// to lower the priority of scan threads
1323
DECLARE_Int32(scan_thread_nice_value);
1324
// Used to modify the recycle interval of tablet schema cache
1325
DECLARE_mInt32(tablet_schema_cache_recycle_interval);
1326
// Granularity is at the column level
1327
DECLARE_mInt32(tablet_schema_cache_capacity);
1328
1329
// Use `LOG(FATAL)` to replace `throw` when true
1330
DECLARE_mBool(exit_on_exception);
1331
1332
// cgroup
1333
DECLARE_String(doris_cgroup_cpu_path);
1334
DECLARE_mBool(enable_be_proc_monitor);
1335
DECLARE_mInt32(be_proc_monitor_interval_ms);
1336
DECLARE_Int32(workload_group_metrics_interval_ms);
1337
1338
DECLARE_mBool(enable_workload_group_memory_gc);
1339
1340
// This config controls whether the s3 file writer would flush cache asynchronously
1341
DECLARE_Bool(enable_flush_file_cache_async);
1342
1343
// Remove predicate that is always true for a segment.
1344
DECLARE_Bool(ignore_always_true_predicate_for_segment);
1345
1346
// Ingest binlog work pool size
1347
DECLARE_Int32(ingest_binlog_work_pool_size);
1348
1349
// Ingest binlog with persistent connection
1350
DECLARE_Bool(enable_ingest_binlog_with_persistent_connection);
1351
1352
// Log ingest binlog elapsed threshold, -1 means no log
1353
DECLARE_mInt64(ingest_binlog_elapsed_threshold_ms);
1354
1355
// Download binlog rate limit, unit is KB/s
1356
DECLARE_Int32(download_binlog_rate_limit_kbs);
1357
1358
DECLARE_mInt32(buffered_reader_read_timeout_ms);
1359
1360
// whether to enable /api/snapshot api
1361
DECLARE_Bool(enable_snapshot_action);
1362
1363
// The max columns size for a tablet schema
1364
DECLARE_mInt32(variant_max_merged_tablet_schema_size);
1365
1366
DECLARE_mInt64(local_exchange_buffer_mem_limit);
1367
1368
DECLARE_mInt64(enable_debug_log_timeout_secs);
1369
1370
DECLARE_mBool(enable_column_type_check);
1371
1372
// Tolerance for the number of partition id 0 in rowset, default 0
1373
DECLARE_Int32(ignore_invalid_partition_id_rowset_num);
1374
1375
DECLARE_mInt32(report_query_statistics_interval_ms);
1376
DECLARE_mInt32(query_statistics_reserve_timeout_ms);
1377
DECLARE_mInt32(report_exec_status_thread_num);
1378
1379
// consider two high usage disk at the same available level if they do not exceed this diff.
1380
DECLARE_mDouble(high_disk_avail_level_diff_usages);
1381
1382
// create tablet in partition random robin idx lru size, default 10000
1383
DECLARE_Int32(partition_disk_index_lru_size);
1384
DECLARE_String(spill_storage_root_path);
1385
// Spill storage limit specified as number of bytes
1386
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
1387
// or percentage of capaity ('<int>%').
1388
// Defaults to bytes if no unit is given.
1389
// Must larger than 0.
1390
// If specified as percentage, the final limit value is:
1391
//   disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit
1392
DECLARE_String(spill_storage_limit);
1393
DECLARE_mInt32(spill_gc_interval_ms);
1394
DECLARE_mInt32(spill_gc_work_time_ms);
1395
DECLARE_Int32(spill_io_thread_pool_thread_num);
1396
DECLARE_Int32(spill_io_thread_pool_queue_size);
1397
DECLARE_Int64(spill_in_paused_queue_timeout_ms);
1398
1399
DECLARE_mBool(check_segment_when_build_rowset_meta);
1400
1401
DECLARE_Int32(num_query_ctx_map_partitions);
1402
1403
DECLARE_mBool(force_azure_blob_global_endpoint);
1404
1405
DECLARE_mBool(enable_s3_rate_limiter);
1406
DECLARE_mInt64(s3_get_bucket_tokens);
1407
DECLARE_mInt64(s3_get_token_per_second);
1408
DECLARE_mInt64(s3_get_token_limit);
1409
1410
DECLARE_mInt64(s3_put_bucket_tokens);
1411
DECLARE_mInt64(s3_put_token_per_second);
1412
DECLARE_mInt64(s3_put_token_limit);
1413
// max s3 client retry times
1414
DECLARE_mInt32(max_s3_client_retry);
1415
// When meet s3 429 error, the "get" request will
1416
// sleep s3_read_base_wait_time_ms (*1, *2, *3, *4) ms
1417
// get try again.
1418
// The max sleep time is s3_read_max_wait_time_ms
1419
// and the max retry time is max_s3_client_retry
1420
DECLARE_mInt32(s3_read_base_wait_time_ms);
1421
DECLARE_mInt32(s3_read_max_wait_time_ms);
1422
1423
// write as inverted index tmp directory
1424
DECLARE_String(tmp_file_dir);
1425
1426
// the directory for storing the trino-connector plugins.
1427
DECLARE_String(trino_connector_plugin_dir);
1428
1429
// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client
1430
DECLARE_mString(ca_cert_file_paths);
1431
1432
/** Table sink configurations(currently contains only external table types) **/
1433
// Minimum data processed to scale writers in exchange when non partition writing
1434
DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold);
1435
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1436
DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold);
1437
// Minimum partition data processed to rebalance writers in exchange when partition writing
1438
DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold);
1439
// Maximum processed partition nums of per writer when partition writing
1440
DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer);
1441
1442
/** Hive sink configurations **/
1443
DECLARE_mInt64(hive_sink_max_file_size);
1444
1445
/** Iceberg sink configurations **/
1446
DECLARE_mInt64(iceberg_sink_max_file_size);
1447
1448
// Number of open tries, default 1 means only try to open once.
1449
// Retry the Open num_retries time waiting 100 milliseconds between retries.
1450
DECLARE_mInt32(thrift_client_open_num_tries);
1451
1452
// http scheme in S3Client to use. E.g. http or https
1453
DECLARE_String(s3_client_http_scheme);
1454
1455
DECLARE_mBool(ignore_schema_change_check);
1456
1457
/** Only use in fuzzy test **/
1458
DECLARE_mInt64(string_overflow_size);
1459
1460
// The min thread num for BufferedReaderPrefetchThreadPool
1461
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread);
1462
// The max thread num for BufferedReaderPrefetchThreadPool
1463
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread);
1464
// The min thread num for S3FileUploadThreadPool
1465
DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread);
1466
// The max thread num for S3FileUploadThreadPool
1467
DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread);
1468
// The maximum jvm heap usage ratio for hdfs write workload
1469
DECLARE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio);
1470
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
1471
DECLARE_mInt64(hdfs_jni_write_sleep_milliseconds);
1472
// The max retry times when hdfs write failed
1473
DECLARE_mInt64(hdfs_jni_write_max_retry_time);
1474
1475
// The min thread num for NonBlockCloseThreadPool
1476
DECLARE_Int64(min_nonblock_close_thread_num);
1477
// The max thread num for NonBlockCloseThreadPool
1478
DECLARE_Int64(max_nonblock_close_thread_num);
1479
// The possibility that mem allocator throws an exception during memory allocation
1480
// This config is for test usage, be careful when changing it.
1481
DECLARE_mDouble(mem_alloc_fault_probability);
1482
// The time out milliseconds for remote fetch schema RPC
1483
DECLARE_mInt64(fetch_remote_schema_rpc_timeout_ms);
1484
// The size of the local buffer for S3FileSytem's upload function
1485
1486
DECLARE_Int64(s3_file_system_local_upload_buffer_size);
1487
1488
//JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues.
1489
DECLARE_Bool(enable_jvm_monitor);
1490
1491
// Num threads to load data dirs, default value -1 indicates the same number of threads as the number of data dirs
1492
DECLARE_Int32(load_data_dirs_threads);
1493
1494
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
1495
DECLARE_mBool(skip_loading_stale_rowset_meta);
1496
// Whether to use file to record log. When starting BE with --console,
1497
// all logs will be written to both standard output and file.
1498
// Disable this option will no longer use file to record log.
1499
// Only works when starting BE with --console.
1500
DECLARE_Bool(enable_file_logger);
1501
1502
// The minimum row group size when exporting Parquet files.
1503
DECLARE_Int64(min_row_group_size);
1504
1505
DECLARE_mInt64(compaction_memory_bytes_limit);
1506
1507
DECLARE_mInt64(compaction_batch_size);
1508
1509
DECLARE_mBool(enable_parquet_page_index);
1510
1511
// Wheather to ignore not found file in external teble(eg, hive)
1512
// Default is true, if set to false, the not found file will result in query failure.
1513
DECLARE_mBool(ignore_not_found_file_in_external_table);
1514
1515
DECLARE_mBool(enable_hdfs_mem_limiter);
1516
1517
// Define how many percent data in hashtable bigger than limit
1518
// we should do agg limit opt
1519
DECLARE_mInt16(topn_agg_limit_multiplier);
1520
1521
DECLARE_mInt64(tablet_meta_serialize_size_limit);
1522
1523
DECLARE_mInt64(pipeline_task_leakage_detect_period_secs);
1524
// To be compatible with hadoop's block compression
1525
DECLARE_mInt32(snappy_compression_block_size);
1526
DECLARE_mInt32(lz4_compression_block_size);
1527
1528
DECLARE_mBool(enable_pipeline_task_leakage_detect);
1529
1530
DECLARE_mInt32(check_score_rounds_num);
1531
1532
// MB
1533
DECLARE_Int32(query_cache_size);
1534
DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
1535
1536
DECLARE_mBool(enable_delete_bitmap_merge_on_compaction);
1537
// Enable validation to check the correctness of table size.
1538
DECLARE_Bool(enable_table_size_correctness_check);
1539
// Enable sleep 5s between delete cumulative compaction.
1540
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
1541
1542
DECLARE_mInt32(compaction_num_per_round);
1543
1544
DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds);
1545
DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
1546
DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
1547
DECLARE_mInt32(schema_dict_cache_capacity);
1548
1549
// whether to prune rows with delete sign = 1 in base compaction
1550
// ATTN: this config is only for test
1551
DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
1552
1553
DECLARE_mBool(enable_mow_verbose_log);
1554
1555
DECLARE_mInt32(tablet_sched_delay_time_ms);
1556
DECLARE_mInt32(load_trigger_compaction_version_percent);
1557
DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
1558
DECLARE_mBool(enable_compaction_pause_on_high_memory);
1559
1560
#ifdef BE_TEST
1561
// test s3
1562
DECLARE_String(test_s3_resource);
1563
DECLARE_String(test_s3_ak);
1564
DECLARE_String(test_s3_sk);
1565
DECLARE_String(test_s3_endpoint);
1566
DECLARE_String(test_s3_region);
1567
DECLARE_String(test_s3_bucket);
1568
DECLARE_String(test_s3_prefix);
1569
#endif
1570
1571
class Register {
1572
public:
1573
    struct Field {
1574
        const char* type = nullptr;
1575
        const char* name = nullptr;
1576
        void* storage = nullptr;
1577
        const char* defval = nullptr;
1578
        bool valmutable = false;
1579
        Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1580
              bool fvalmutable)
1581
                : type(ftype),
1582
                  name(fname),
1583
                  storage(fstorage),
1584
                  defval(fdefval),
1585
661
                  valmutable(fvalmutable) {}
1586
    };
1587
1588
public:
1589
    static std::map<std::string, Field>* _s_field_map;
1590
1591
public:
1592
    Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1593
657
             bool fvalmutable) {
1594
657
        if (_s_field_map == nullptr) {
1595
1
            _s_field_map = new std::map<std::string, Field>();
1596
1
        }
1597
657
        Field field(ftype, fname, fstorage, fdefval, fvalmutable);
1598
657
        _s_field_map->insert(std::make_pair(std::string(fname), field));
1599
657
    }
1600
};
1601
1602
// RegisterConfValidator class is used to store validator function of registered config fields in
1603
// Register::_s_field_map.
1604
// If any validator return false when BE bootstart, the bootstart will be terminated.
1605
// If validator return false when use http API to update some config, the config will not
1606
// be modified and the API will return failure.
1607
class RegisterConfValidator {
1608
public:
1609
    // Validator for each config name.
1610
    static std::map<std::string, std::function<bool()>>* _s_field_validator;
1611
1612
public:
1613
    RegisterConfValidator(const char* fname, const std::function<bool()>& validator) {
1614
        if (_s_field_validator == nullptr) {
1615
            _s_field_validator = new std::map<std::string, std::function<bool()>>();
1616
        }
1617
        // register validator to _s_field_validator
1618
        _s_field_validator->insert(std::make_pair(std::string(fname), validator));
1619
    }
1620
};
1621
1622
// configuration properties load from config file.
1623
class Properties {
1624
public:
1625
    // load conf from file, if must_exist is true and file does not exist, return false
1626
    bool load(const char* conf_file, bool must_exist = true);
1627
1628
    // Find the config value by key from `file_conf_map`.
1629
    // If found, set `retval` to the config value,
1630
    // or set `retval` to `defstr`
1631
    // if retval is not set(in case defstr is nullptr), set is_retval_set to false
1632
    template <typename T>
1633
    bool get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set,
1634
                        std::string& rawval) const;
1635
1636
    void set(const std::string& key, const std::string& val);
1637
1638
    void set_force(const std::string& key, const std::string& val);
1639
1640
    // dump props to conf file
1641
    Status dump(const std::string& conffile);
1642
1643
private:
1644
    std::map<std::string, std::string> file_conf_map;
1645
};
1646
1647
// full configurations.
1648
extern std::map<std::string, std::string>* full_conf_map;
1649
1650
extern std::mutex custom_conf_lock;
1651
1652
// Init the config from `conf_file`.
1653
// If fill_conf_map is true, the updated config will also update the `full_conf_map`.
1654
// If must_exist is true and `conf_file` does not exist, this function will return false.
1655
// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
1656
bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist = true,
1657
          bool set_to_default = true);
1658
1659
Status set_config(const std::string& field, const std::string& value, bool need_persist = false,
1660
                  bool force = false);
1661
1662
Status persist_config(const std::string& field, const std::string& value);
1663
1664
std::mutex* get_mutable_string_config_lock();
1665
1666
std::vector<std::vector<std::string>> get_config_info();
1667
1668
Status set_fuzzy_configs();
1669
1670
void update_config(const std::string& field, const std::string& value);
1671
1672
} // namespace config
1673
} // namespace doris