Coverage Report

Created: 2025-07-28 21:04

/root/doris/be/src/common/config.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <cstdint>
21
#include <functional>
22
#include <map>
23
#include <mutex>
24
#include <string>
25
#include <utility>
26
#include <vector>
27
28
#define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME
29
30
#define DECLARE_Bool(name) DECLARE_FIELD(bool, name)
31
#define DECLARE_Int16(name) DECLARE_FIELD(int16_t, name)
32
#define DECLARE_Int32(name) DECLARE_FIELD(int32_t, name)
33
#define DECLARE_Int64(name) DECLARE_FIELD(int64_t, name)
34
#define DECLARE_Double(name) DECLARE_FIELD(double, name)
35
#define DECLARE_String(name) DECLARE_FIELD(std::string, name)
36
#define DECLARE_Bools(name) DECLARE_FIELD(std::vector<bool>, name)
37
#define DECLARE_Int16s(name) DECLARE_FIELD(std::vector<int16_t>, name)
38
#define DECLARE_Int32s(name) DECLARE_FIELD(std::vector<int32_t>, name)
39
#define DECLARE_Int64s(name) DECLARE_FIELD(std::vector<int64_t>, name)
40
#define DECLARE_Doubles(name) DECLARE_FIELD(std::vector<double>, name)
41
#define DECLARE_Strings(name) DECLARE_FIELD(std::vector<std::string>, name)
42
#define DECLARE_mBool(name) DECLARE_FIELD(bool, name)
43
#define DECLARE_mInt16(name) DECLARE_FIELD(int16_t, name)
44
#define DECLARE_mInt32(name) DECLARE_FIELD(int32_t, name)
45
#define DECLARE_mInt64(name) DECLARE_FIELD(int64_t, name)
46
#define DECLARE_mDouble(name) DECLARE_FIELD(double, name)
47
#define DECLARE_mString(name) DECLARE_FIELD(std::string, name)
48
49
#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE)                      \
50
21
    FIELD_TYPE FIELD_NAME;                                                                   \
51
21
    static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &(FIELD_NAME), FIELD_DEFAULT, \
52
21
                                     VALMUTABLE);
53
54
#define DEFINE_VALIDATOR(FIELD_NAME, VALIDATOR)              \
55
    static auto validator_##FIELD_NAME = VALIDATOR;          \
56
    static RegisterConfValidator reg_validator_##FIELD_NAME( \
57
            #FIELD_NAME, []() -> bool { return validator_##FIELD_NAME(FIELD_NAME); });
58
59
1
#define DEFINE_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false)
60
1
#define DEFINE_Bools(name, defaultstr) DEFINE_FIELD(std::vector<bool>, name, defaultstr, false)
61
1
#define DEFINE_Doubles(name, defaultstr) DEFINE_FIELD(std::vector<double>, name, defaultstr, false)
62
1
#define DEFINE_Int16s(name, defaultstr) DEFINE_FIELD(std::vector<int16_t>, name, defaultstr, false)
63
1
#define DEFINE_Int32s(name, defaultstr) DEFINE_FIELD(std::vector<int32_t>, name, defaultstr, false)
64
1
#define DEFINE_Int64s(name, defaultstr) DEFINE_FIELD(std::vector<int64_t>, name, defaultstr, false)
65
3
#define DEFINE_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false)
66
1
#define DEFINE_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false)
67
1
#define DEFINE_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false)
68
1
#define DEFINE_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false)
69
2
#define DEFINE_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false)
70
#define DEFINE_Strings(name, defaultstr) \
71
1
    DEFINE_FIELD(std::vector<std::string>, name, defaultstr, false)
72
1
#define DEFINE_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true)
73
1
#define DEFINE_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true)
74
1
#define DEFINE_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true)
75
1
#define DEFINE_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true)
76
1
#define DEFINE_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true)
77
1
#define DEFINE_mString(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, true)
78
#define DEFINE_Validator(name, validator) DEFINE_VALIDATOR(name, validator)
79
80
namespace doris {
81
class Status;
82
83
// If you want to modify the value of config, please go to common/config.cpp to modify.
84
namespace config {
85
// Dir of custom config file
86
DECLARE_String(custom_config_dir);
87
88
// Dir of jdbc drivers
89
DECLARE_String(jdbc_drivers_dir);
90
91
// cluster id
92
DECLARE_Int32(cluster_id);
93
// port on which BackendService is exported
94
DECLARE_Int32(be_port);
95
96
// port for brpc
97
DECLARE_Int32(brpc_port);
98
99
// port for arrow flight sql
100
// Default -1, do not start arrow flight sql server.
101
DECLARE_Int32(arrow_flight_sql_port);
102
103
// If the external client cannot directly access priority_networks, set public_host to be accessible
104
// to external client.
105
// There are usually two usage scenarios:
106
// 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network.
107
// However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be
108
// randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx.
109
// 2. if priority_networks is an internal network IP, and BE node has its own independent external IP,
110
// but Doris currently does not support modifying priority_networks, setting public_host to the real external IP.
111
DECLARE_mString(public_host);
112
113
// If the BE node is connected to the external network through a reverse proxy like Nginx
114
// and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy
115
// `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example:
116
// upstream arrowflight {
117
//    server 10.16.10.8:8069;
118
//    server 10.16.10.8:8068;
119
//}
120
// server {
121
//    listen 8167 http2;
122
//    listen [::]:8167 http2;
123
//    server_name doris.arrowflight.com;
124
// }
125
DECLARE_Int32(arrow_flight_sql_proxy_port);
126
127
// the number of bthreads for brpc, the default value is set to -1,
128
// which means the number of bthreads is #cpu-cores
129
DECLARE_Int32(brpc_num_threads);
130
DECLARE_Int32(brpc_idle_timeout_sec);
131
132
// Declare a selection strategy for those servers have many ips.
133
// Note that there should at most one ip match this list.
134
// This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
135
// If no ip match this rule, will choose one randomly.
136
DECLARE_String(priority_networks);
137
138
// performance moderate or compact, only tcmalloc compile
139
DECLARE_String(memory_mode);
140
141
// if true, process memory limit and memory usage based on cgroup memory info.
142
DECLARE_mBool(enable_use_cgroup_memory_info);
143
144
// process memory limit specified as number of bytes
145
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
146
// or percentage of the physical memory ('<int>%').
147
// defaults to bytes if no unit is given"
148
// must larger than 0. and if larger than physical memory size,
149
// it will be set to physical memory size.
150
DECLARE_String(mem_limit);
151
152
// Soft memory limit as a fraction of hard memory limit.
153
DECLARE_Double(soft_mem_limit_frac);
154
155
// Cache capacity reduce mem limit as a fraction of soft mem limit.
156
DECLARE_mDouble(cache_capacity_reduce_mem_limit_frac);
157
158
// Schema change memory limit as a fraction of soft memory limit.
159
DECLARE_Double(schema_change_mem_limit_frac);
160
161
// Many modern allocators (for example) do not do a mremap for
162
// realloc, even in case of large enough chunks of memory. Although this allows
163
// you to increase performance and reduce memory consumption during realloc.
164
// To fix this, we do mremap manually if the chunk of memory is large enough.
165
//
166
// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
167
// space is very slow, especially in the case of a large number of threads. We
168
// expect that the set of operations mmap/something to do/mremap can only be
169
// performed about 1000 times per second.
170
//
171
// P.S. This is also required, because tcmalloc can not allocate a chunk of
172
// memory greater than 16 GB.
173
DECLARE_mInt64(mmap_threshold); // bytes
174
175
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
176
// Increase can reduce the number of hash table resize, but may waste more memory.
177
DECLARE_mInt32(hash_table_double_grow_degree);
178
179
// The max fill rate for hash table
180
DECLARE_mInt32(max_fill_rate);
181
182
DECLARE_mInt32(double_resize_threshold);
183
184
// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default -1.
185
// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%), which is 3.2G on a 64G machine.
186
// Turn up max. more memory buffers will be reserved for Memory GC.
187
// Turn down max. will use as much memory as possible.
188
// note that: `max_` prefix should be removed, but keep it for compatibility.
189
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
190
191
// reserve a small amount of memory so we do not trigger MinorGC
192
DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
193
194
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
195
DECLARE_mString(process_minor_gc_size);
196
DECLARE_mString(process_full_gc_size);
197
198
// If true, when the process does not exceed the soft mem limit, the query memory will not be limited;
199
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
200
// used memory and the exec_mem_limit will be canceled.
201
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
202
DECLARE_mBool(enable_query_memory_overcommit);
203
204
// gc will release cache, cancel task, and task will wait for gc to release memory,
205
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
206
DECLARE_mBool(disable_memory_gc);
207
208
// if false, turn off all stacktrace
209
DECLARE_mBool(enable_stacktrace);
210
211
// when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G,
212
// if alloc successful, will print a warning with stacktrace, but not prevent memory alloc.
213
// if alloc failed using Doris Allocator, will print stacktrace in error log.
214
// if is -1, disable print stacktrace when alloc large memory.
215
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
216
217
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled.
218
// if you need a core dump to analyze large memory allocation,
219
// modify this parameter to crash when large memory allocation occur will help
220
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
221
222
// The actual meaning of this parameter is `debug_memory`.
223
// 1. crash in memory tracker inaccurate, if memory tracker value is inaccurate, BE will crash.
224
//    usually used in test environments, default value is false.
225
// 2. print more memory logs.
226
DECLARE_mBool(crash_in_memory_tracker_inaccurate);
227
228
// default is true. if any memory tracking in Orphan mem tracker will report error.
229
// !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time.
230
// allocator free memory not need to check, because when the thread memory tracker label is Orphan,
231
// use the tracker saved in Allocator.
232
DECLARE_mBool(enable_memory_orphan_check);
233
234
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
235
DECLARE_mInt32(thread_wait_gc_max_milliseconds);
236
237
// reach mem limit, don't serialize in batch
238
DECLARE_mInt64(pre_serialize_keys_limit_bytes);
239
240
// the port heartbeat service used
241
DECLARE_Int32(heartbeat_service_port);
242
// the count of heart beat service
243
DECLARE_Int32(heartbeat_service_thread_count);
244
// the count of thread to create table
245
DECLARE_Int32(create_tablet_worker_count);
246
// the count of thread to drop table
247
DECLARE_Int32(drop_tablet_worker_count);
248
// the count of thread to batch load
249
DECLARE_Int32(push_worker_count_normal_priority);
250
// the count of thread to high priority batch load
251
DECLARE_Int32(push_worker_count_high_priority);
252
// the count of thread to publish version
253
DECLARE_Int32(publish_version_worker_count);
254
// the count of tablet thread to publish version
255
DECLARE_Int32(tablet_publish_txn_max_thread);
256
// the timeout of EnginPublishVersionTask
257
DECLARE_Int32(publish_version_task_timeout_s);
258
// the count of thread to calc delete bitmap
259
DECLARE_Int32(calc_delete_bitmap_max_thread);
260
// the count of thread to calc delete bitmap worker, only used for cloud
261
DECLARE_Int32(calc_delete_bitmap_worker_count);
262
// the count of thread to calc tablet delete bitmap task, only used for cloud
263
DECLARE_Int32(calc_tablet_delete_bitmap_task_max_thread);
264
// the count of thread to clear transaction task
265
DECLARE_Int32(clear_transaction_task_worker_count);
266
// the count of thread to delete
267
DECLARE_Int32(delete_worker_count);
268
// the count of thread to alter table
269
DECLARE_Int32(alter_tablet_worker_count);
270
// the count of thread to alter index
271
DECLARE_Int32(alter_index_worker_count);
272
// the count of thread to clone
273
DECLARE_Int32(clone_worker_count);
274
// the count of thread to clone
275
DECLARE_Int32(storage_medium_migrate_count);
276
// the count of thread to check consistency
277
DECLARE_Int32(check_consistency_worker_count);
278
// the count of thread to upload
279
DECLARE_Int32(upload_worker_count);
280
// the count of thread to download
281
DECLARE_Int32(download_worker_count);
282
// the count of thread to make snapshot
283
DECLARE_Int32(make_snapshot_worker_count);
284
// the count of thread to release snapshot
285
DECLARE_Int32(release_snapshot_worker_count);
286
// report random wait a little time to avoid FE receiving multiple be reports at the same time.
287
// do not set it to false for production environment
288
DECLARE_mBool(report_random_wait);
289
// the interval time(seconds) for agent report tasks signature to FE
290
DECLARE_mInt32(report_task_interval_seconds);
291
// the interval time(seconds) for refresh storage policy from FE
292
DECLARE_mInt32(storage_refresh_storage_policy_task_interval_seconds);
293
// the interval time(seconds) for agent report disk state to FE
294
DECLARE_mInt32(report_disk_state_interval_seconds);
295
// the interval time(seconds) for agent report olap table to FE
296
DECLARE_mInt32(report_tablet_interval_seconds);
297
// the max download speed(KB/s)
298
DECLARE_mInt32(max_download_speed_kbps);
299
// download low speed limit(KB/s)
300
DECLARE_mInt32(download_low_speed_limit_kbps);
301
// download low speed time(seconds)
302
DECLARE_mInt32(download_low_speed_time);
303
// whether to download small files in batch.
304
DECLARE_mBool(enable_batch_download);
305
// whether to check md5sum when download
306
DECLARE_mBool(enable_download_md5sum_check);
307
// download binlog meta timeout
308
DECLARE_mInt32(download_binlog_meta_timeout_ms);
309
// the interval time(seconds) for agent report index policy to FE
310
DECLARE_mInt32(report_index_policy_interval_seconds);
311
312
// deprecated, use env var LOG_DIR in be.conf
313
DECLARE_String(sys_log_dir);
314
// for udf
315
DECLARE_String(user_function_dir);
316
// INFO, WARNING, ERROR, FATAL
317
DECLARE_String(sys_log_level);
318
// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
319
DECLARE_String(sys_log_roll_mode);
320
// log roll num
321
DECLARE_Int32(sys_log_roll_num);
322
// verbose log
323
DECLARE_Strings(sys_log_verbose_modules);
324
// verbose log level
325
DECLARE_Int32(sys_log_verbose_level);
326
// verbose log FLAGS_v
327
DECLARE_Int32(sys_log_verbose_flags_v);
328
// log buffer level
329
DECLARE_String(log_buffer_level);
330
// log enable custom date time format
331
DECLARE_Bool(sys_log_enable_custom_date_time_format);
332
// log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time)
333
DECLARE_String(sys_log_custom_date_time_format);
334
// log custom date time milliseconds format (fmt::format)
335
DECLARE_String(sys_log_custom_date_time_ms_format);
336
337
// number of threads available to serve backend execution requests
338
DECLARE_Int32(be_service_threads);
339
340
// interval between profile reports; in seconds
341
DECLARE_mInt32(status_report_interval);
342
DECLARE_mInt32(pipeline_status_report_interval);
343
// if true, each disk will have a separate thread pool for scanner
344
DECLARE_Bool(doris_enable_scanner_thread_pool_per_disk);
345
// the timeout of a work thread to wait the blocking priority queue to get a task
346
DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
347
// number of scanner thread pool size for olap table
348
// and the min thread num of remote scanner thread pool
349
DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
350
DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num);
351
// number of batch size to fetch the remote split source
352
DECLARE_mInt32(remote_split_source_batch_size);
353
// max number of remote scanner thread pool size
354
// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
355
DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
356
// number of olap scanner thread pool queue size
357
DECLARE_Int32(doris_scanner_thread_pool_queue_size);
358
// default thrift client connect timeout(in seconds)
359
DECLARE_mInt32(thrift_connect_timeout_seconds);
360
// default thrift client retry interval (in milliseconds)
361
DECLARE_mInt64(thrift_client_retry_interval_ms);
362
// max message size of thrift request
363
// default: 100 * 1024 * 1024
364
DECLARE_mInt64(thrift_max_message_size);
365
// max row count number for single scan range, used in segmentv1
366
DECLARE_mInt32(doris_scan_range_row_count);
367
// max bytes number for single scan range, used in segmentv2
368
DECLARE_mInt32(doris_scan_range_max_mb);
369
// max bytes number for single scan block, used in segmentv2
370
DECLARE_mInt32(doris_scan_block_max_mb);
371
// single read execute fragment row number
372
DECLARE_mInt32(doris_scanner_row_num);
373
// single read execute fragment row bytes
374
DECLARE_mInt32(doris_scanner_row_bytes);
375
// single read execute fragment max run time millseconds
376
DECLARE_mInt32(doris_scanner_max_run_time_ms);
377
DECLARE_mInt32(min_bytes_in_scanner_queue);
378
// (Advanced) Maximum size of per-query receive-side buffer
379
DECLARE_mInt32(exchg_node_buffer_size_bytes);
380
DECLARE_mInt32(exchg_buffer_queue_capacity_factor);
381
382
DECLARE_mInt64(column_dictionary_key_ratio_threshold);
383
DECLARE_mInt64(column_dictionary_key_size_threshold);
384
// memory_limitation_per_thread_for_schema_change_bytes unit bytes
385
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
386
DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);
387
388
// all cache prune interval, used by GC and periodic thread.
389
DECLARE_mInt32(cache_prune_interval_sec);
390
DECLARE_mInt32(cache_periodic_prune_stale_sweep_sec);
391
// the clean interval of tablet lookup cache
392
DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec);
393
DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec);
394
DECLARE_mInt32(disk_stat_monitor_interval);
395
DECLARE_mInt32(unused_rowset_monitor_interval);
396
DECLARE_mInt32(quering_rowsets_evict_interval);
397
DECLARE_String(storage_root_path);
398
DECLARE_mString(broken_storage_path);
399
400
// Config is used to check incompatible old format hdr_ format
401
// whether doris uses strict way. When config is true, process will log fatal
402
// and exit. When config is false, process will only log warning.
403
DECLARE_Bool(storage_strict_check_incompatible_old_format);
404
405
// BE process will exit if the percentage of error disk reach this value.
406
DECLARE_mInt32(max_percentage_of_error_disk);
407
DECLARE_mInt32(default_num_rows_per_column_file_block);
408
// pending data policy
409
DECLARE_mInt32(pending_data_expire_time_sec);
410
// inc_rowset snapshot rs sweep time interval
411
DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec);
412
// tablet stale rowset sweep by threshold size
413
DECLARE_Bool(tablet_rowset_stale_sweep_by_size);
414
DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size);
415
// garbage sweep policy
416
DECLARE_Int32(max_garbage_sweep_interval);
417
DECLARE_Int32(min_garbage_sweep_interval);
418
// garbage sweep every batch will sleep 1ms
419
DECLARE_mInt32(garbage_sweep_batch_size);
420
DECLARE_mInt32(snapshot_expire_time_sec);
421
// It is only a recommended value. When the disk space is insufficient,
422
// the file storage period under trash dose not have to comply with this parameter.
423
DECLARE_mInt32(trash_file_expire_time_sec);
424
// minimum file descriptor number
425
// modify them upon necessity
426
DECLARE_Int32(min_file_descriptor_number);
427
DECLARE_mBool(disable_segment_cache);
428
DECLARE_String(row_cache_mem_limit);
429
430
// Cache for storage page size
431
DECLARE_String(storage_page_cache_limit);
432
// Shard size for page cache, the value must be power of two.
433
// It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions.
434
DECLARE_Int32(storage_page_cache_shard_size);
435
// Percentage for index page cache
436
// all storage page cache will be divided into data_page_cache and index_page_cache
437
DECLARE_Int32(index_page_cache_percentage);
438
// whether to disable page cache feature in storage
439
// TODO delete it. Divided into Data page, Index page, pk index page
440
DECLARE_Bool(disable_storage_page_cache);
441
// whether to disable row cache feature in storage
442
DECLARE_mBool(disable_storage_row_cache);
443
// whether to disable pk page cache feature in storage
444
DECLARE_Bool(disable_pk_storage_page_cache);
445
DECLARE_Bool(enable_non_pipeline);
446
447
// Cache for mow primary key storage page size, it's seperated from
448
// storage_page_cache_limit
449
DECLARE_String(pk_storage_page_cache_limit);
450
// data page size for primary key index
451
DECLARE_Int32(primary_key_data_page_size);
452
453
// inc_rowset snapshot rs sweep time interval
454
DECLARE_mInt32(data_page_cache_stale_sweep_time_sec);
455
DECLARE_mInt32(index_page_cache_stale_sweep_time_sec);
456
// great impact on the performance of MOW, so it can be longer.
457
DECLARE_mInt32(pk_index_page_cache_stale_sweep_time_sec);
458
459
DECLARE_Bool(enable_low_cardinality_optimize);
460
DECLARE_Bool(enable_low_cardinality_cache_code);
461
462
// be policy
463
// whether check compaction checksum
464
DECLARE_mBool(enable_compaction_checksum);
465
// whether disable automatic compaction task
466
DECLARE_mBool(disable_auto_compaction);
467
// whether enable vertical compaction
468
DECLARE_mBool(enable_vertical_compaction);
469
// whether enable ordered data compaction
470
DECLARE_mBool(enable_ordered_data_compaction);
471
// In vertical compaction, column number for every group
472
DECLARE_mInt32(vertical_compaction_num_columns_per_group);
473
// In vertical compaction, max memory usage for row_source_buffer
474
DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
475
// In vertical compaction, max dest segment file size
476
DECLARE_mInt64(vertical_compaction_max_segment_size);
477
478
// If enabled, segments will be flushed column by column
479
DECLARE_mBool(enable_vertical_segment_writer);
480
481
// In ordered data compaction, min segment size for input rowset
482
DECLARE_mInt32(ordered_data_compaction_min_segment_size);
483
484
// This config can be set to limit thread number in compaction thread pool.
485
DECLARE_mInt32(max_base_compaction_threads);
486
DECLARE_mInt32(max_cumu_compaction_threads);
487
DECLARE_mInt32(max_single_replica_compaction_threads);
488
489
DECLARE_Bool(enable_base_compaction_idle_sched);
490
DECLARE_mInt64(base_compaction_min_rowset_num);
491
DECLARE_mInt64(base_compaction_max_compaction_score);
492
DECLARE_mInt64(mow_base_compaction_max_compaction_score);
493
DECLARE_mDouble(base_compaction_min_data_ratio);
494
DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes);
495
496
DECLARE_Bool(enable_skip_tablet_compaction);
497
DECLARE_mInt32(skip_tablet_compaction_second);
498
// output rowset of cumulative compaction total disk size exceed this config size,
499
// this rowset will be given to base compaction, unit is m byte.
500
DECLARE_mInt64(compaction_promotion_size_mbytes);
501
502
// output rowset of cumulative compaction total disk size exceed this config ratio of
503
// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
504
// 0 and 1.
505
DECLARE_mDouble(compaction_promotion_ratio);
506
507
// the smallest size of rowset promotion. When the rowset is less than this config, this
508
// rowset will be not given to base compaction. The unit is m byte.
509
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
510
511
// When output rowset of cumulative compaction total version count (end_version - start_version)
512
// exceed this config count, the rowset will be moved to base compaction
513
// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
514
// related cost on delete bitmap more effectively.
515
DECLARE_mInt64(compaction_promotion_version_count);
516
517
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
518
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
519
DECLARE_mInt64(compaction_min_size_mbytes);
520
521
// cumulative compaction policy: min and max delta file's number
522
DECLARE_mInt64(cumulative_compaction_min_deltas);
523
DECLARE_mInt64(cumulative_compaction_max_deltas);
524
DECLARE_mInt32(cumulative_compaction_max_deltas_factor);
525
526
// This config can be set to limit thread number in  multiget thread pool.
527
DECLARE_mInt32(multi_get_max_threads);
528
529
// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
530
DECLARE_mInt64(total_permits_for_compaction_score);
531
532
// sleep interval in ms after generated compaction tasks
533
DECLARE_mInt32(generate_compaction_tasks_interval_ms);
534
// sleep interval in second after update replica infos
535
DECLARE_mInt32(update_replica_infos_interval_seconds);
536
537
// Compaction task number per disk.
538
// Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each.
539
DECLARE_mInt32(compaction_task_num_per_disk);
540
// compaction thread num for fast disk(typically .SSD), must be greater than 2.
541
DECLARE_mInt32(compaction_task_num_per_fast_disk);
542
543
// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
544
DECLARE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round);
545
546
// Minimum number of threads required in the thread pool to activate the large cumu compaction delay strategy.
547
// The delay strategy is only applied when the thread pool has at least this many threads.
548
DECLARE_mInt32(large_cumu_compaction_task_min_thread_num);
549
// Maximum size threshold (in bytes) for input rowsets. Compaction tasks with input size
550
// exceeding this threshold will be delayed when thread pool is near capacity. Default 100MB.
551
DECLARE_mInt32(large_cumu_compaction_task_bytes_threshold);
552
// Maximum row count threshold for compaction input. Compaction tasks with row count
553
// exceeding this threshold will be delayed when thread pool is near capacity. Default 1 million.
554
DECLARE_mInt32(large_cumu_compaction_task_row_num_threshold);
555
556
// Not compact the invisible versions, but with some limitations:
557
// if not timeout, keep no more than compaction_keep_invisible_version_max_count versions;
558
// if timeout, keep no more than compaction_keep_invisible_version_min_count versions.
559
DECLARE_mInt32(compaction_keep_invisible_version_timeout_sec);
560
DECLARE_mInt32(compaction_keep_invisible_version_min_count);
561
DECLARE_mInt32(compaction_keep_invisible_version_max_count);
562
563
// Threshold to logging compaction trace, in seconds.
564
DECLARE_mInt32(base_compaction_trace_threshold);
565
DECLARE_mInt32(cumulative_compaction_trace_threshold);
566
DECLARE_mBool(disable_compaction_trace_log);
567
568
// Interval to picking rowset to compact, in seconds
569
DECLARE_mInt64(pick_rowset_to_compact_interval_sec);
570
571
// Compaction priority schedule
572
DECLARE_mBool(enable_compaction_priority_scheduling);
573
DECLARE_mInt32(low_priority_compaction_task_num_per_disk);
574
DECLARE_mInt32(low_priority_compaction_score_threshold);
575
576
// Thread count to do tablet meta checkpoint, -1 means use the data directories count.
577
DECLARE_Int32(max_meta_checkpoint_threads);
578
579
// Threshold to logging agent task trace, in seconds.
580
DECLARE_mInt32(agent_task_trace_threshold_sec);
581
582
// This config can be set to limit thread number in tablet migration thread pool.
583
DECLARE_Int32(min_tablet_migration_threads);
584
DECLARE_Int32(max_tablet_migration_threads);
585
586
DECLARE_mInt32(finished_migration_tasks_size);
587
// If size less than this, the remaining rowsets will be force to complete
588
DECLARE_mInt32(migration_remaining_size_threshold_mb);
589
// If the task runs longer than this time, the task will be terminated, in seconds.
590
// timeout = std::max(migration_task_timeout_secs,  tablet size / 1MB/s)
591
DECLARE_mInt32(migration_task_timeout_secs);
592
// timeout for try_lock migration lock
593
DECLARE_Int64(migration_lock_timeout_ms);
594
595
// Port to start debug webserver on
596
DECLARE_Int32(webserver_port);
597
// Https enable flag
598
DECLARE_Bool(enable_https);
599
// Path of certificate
600
DECLARE_String(ssl_certificate_path);
601
// Path of private key
602
DECLARE_String(ssl_private_key_path);
603
// Whether to check authorization
604
DECLARE_Bool(enable_all_http_auth);
605
// Number of webserver workers
606
DECLARE_Int32(webserver_num_workers);
607
608
DECLARE_Bool(enable_single_replica_load);
609
// Number of download workers for single replica load
610
DECLARE_Int32(single_replica_load_download_num_workers);
611
612
// Used for mini Load. mini load data file will be removed after this time.
613
DECLARE_Int64(load_data_reserve_hours);
614
// log error log will be removed after this time
615
DECLARE_mInt64(load_error_log_reserve_hours);
616
// error log size limit, default 200MB
617
DECLARE_mInt64(load_error_log_limit_bytes);
618
619
// be brpc interface is classified into two categories: light and heavy
620
// each category has diffrent thread number
621
// threads to handle heavy api interface, such as transmit_data/transmit_block etc
622
DECLARE_Int32(brpc_heavy_work_pool_threads);
623
// threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start
624
DECLARE_Int32(brpc_light_work_pool_threads);
625
DECLARE_Int32(brpc_heavy_work_pool_max_queue_size);
626
DECLARE_Int32(brpc_light_work_pool_max_queue_size);
627
DECLARE_mBool(enable_bthread_transmit_block);
628
DECLARE_Int32(brpc_arrow_flight_work_pool_threads);
629
DECLARE_Int32(brpc_arrow_flight_work_pool_max_queue_size);
630
631
// The maximum amount of data that can be processed by a stream load
632
DECLARE_mInt64(streaming_load_max_mb);
633
// Some data formats, such as JSON, cannot be streamed.
634
// Therefore, it is necessary to limit the maximum number of
635
// such data when using stream load to prevent excessive memory consumption.
636
DECLARE_mInt64(streaming_load_json_max_mb);
637
// the alive time of a TabletsChannel.
638
// If the channel does not receive any data till this time,
639
// the channel will be removed.
640
DECLARE_mInt32(streaming_load_rpc_max_alive_time_sec);
641
// the timeout of a rpc to open the tablet writer in remote BE.
642
// short operation time, can set a short timeout
643
DECLARE_Int32(tablet_writer_open_rpc_timeout_sec);
644
// You can ignore brpc error '[E1011]The server is overcrowded' when writing data.
645
DECLARE_mBool(tablet_writer_ignore_eovercrowded);
646
DECLARE_mInt32(slave_replica_writer_rpc_timeout_sec);
647
// Whether to enable stream load record function, the default is false.
648
// False: disable stream load record
649
DECLARE_mBool(enable_stream_load_record);
650
// batch size of stream load record reported to FE
651
DECLARE_mInt32(stream_load_record_batch_size);
652
// expire time of stream load record in rocksdb.
653
DECLARE_Int32(stream_load_record_expire_time_secs);
654
// time interval to clean expired stream load records
655
DECLARE_mInt64(clean_stream_load_record_interval_secs);
656
// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
657
DECLARE_mBool(enable_stream_load_commit_txn_on_be);
658
// The buffer size to store stream table function schema info
659
DECLARE_Int64(stream_tvf_buffer_size);
660
661
// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
662
// You may need to lower the speed when the sink receiver bes are too busy.
663
DECLARE_mInt32(olap_table_sink_send_interval_microseconds);
664
// For auto partition, the send interval will multiply the factor
665
DECLARE_mDouble(olap_table_sink_send_interval_auto_partition_factor);
666
667
// Fragment thread pool
668
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_min);
669
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_max);
670
DECLARE_Int32(fragment_mgr_asynic_work_pool_queue_size);
671
672
// Control the number of disks on the machine.  If 0, this comes from the system settings.
673
DECLARE_Int32(num_disks);
674
// The maximum number of the threads per disk is also the max queue depth per disk.
675
DECLARE_Int32(num_threads_per_disk);
676
// The read size is the size of the reads sent to os.
677
// There is a trade off of latency and throughout, trying to keep disks busy but
678
// not introduce seeks.  The literature seems to agree that with 8 MB reads, random
679
// io and sequential io perform similarly.
680
DECLARE_Int32(read_size);       // 8 * 1024 * 1024, Read Size (in bytes)
681
DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
682
683
// for pprof
684
DECLARE_String(pprof_profile_dir);
685
// for jeprofile in jemalloc
686
DECLARE_mString(jeprofile_dir);
687
// Purge all unused dirty pages for all arenas.
688
DECLARE_mBool(enable_je_purge_dirty_pages);
689
// Purge all unused Jemalloc dirty pages for all arenas when exceed je_dirty_pages_mem_limit and process exceed soft limit.
690
DECLARE_mString(je_dirty_pages_mem_limit_percent);
691
692
// to forward compatibility, will be removed later
693
DECLARE_mBool(enable_token_check);
694
695
// to open/close system metrics
696
DECLARE_Bool(enable_system_metrics);
697
698
// Number of cores Doris will used, this will effect only when it's greater than 0.
699
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
700
DECLARE_Int32(num_cores);
701
702
// When BE start, If there is a broken disk, BE process will exit by default.
703
// Otherwise, we will ignore the broken disk,
704
DECLARE_Bool(ignore_broken_disk);
705
DECLARE_Bool(ignore_file_cache_dir_upgrade_failure);
706
707
// Sleep time in milliseconds between memory maintenance iterations
708
DECLARE_mInt32(memory_maintenance_sleep_time_ms);
709
710
// After full gc, no longer full gc and minor gc during sleep.
711
// After minor gc, no minor gc during sleep, but full gc is possible.
712
DECLARE_mInt32(memory_gc_sleep_time_ms);
713
714
// max write buffer size before flush, default 200MB
715
DECLARE_mInt64(write_buffer_size);
716
// max buffer size used in memtable for the aggregated table, default 400MB
717
DECLARE_mInt64(write_buffer_size_for_agg);
718
719
DECLARE_mInt64(min_write_buffer_size_for_partial_update);
720
// max parallel flush task per memtable writer
721
DECLARE_mInt32(memtable_flush_running_count_limit);
722
723
DECLARE_Int32(load_process_max_memory_limit_percent); // 50%
724
725
// If the memory consumption of load jobs exceed load_process_max_memory_limit,
726
// all load jobs will hang there to wait for memtable flush. We should have a
727
// soft limit which can trigger the memtable flush for the load channel who
728
// consumes lagest memory size before we reach the hard limit. The soft limit
729
// might avoid all load jobs hang at the same time.
730
DECLARE_Int32(load_process_soft_mem_limit_percent);
731
732
// If load memory consumption is within load_process_safe_mem_permit_percent,
733
// memtable memory limiter will do nothing.
734
DECLARE_Int32(load_process_safe_mem_permit_percent);
735
736
// result buffer cancelled time (unit: second)
737
DECLARE_mInt32(result_buffer_cancelled_interval_time);
738
739
// arrow flight result sink buffer rows size, default 4096 * 8
740
DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows);
741
// The timeout for ADBC Client to wait for data using arrow flight reader.
742
// If the query is very complex and no result is generated after this time, consider increasing this timeout.
743
DECLARE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms);
744
745
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
746
DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency);
747
748
// sync tablet_meta when modifying meta
749
DECLARE_mBool(sync_tablet_meta);
750
751
// sync a file writer when it is closed
752
DECLARE_mBool(sync_file_on_close);
753
754
// default thrift rpc timeout ms
755
DECLARE_mInt32(thrift_rpc_timeout_ms);
756
757
// txn commit rpc timeout
758
DECLARE_mInt32(txn_commit_rpc_timeout_ms);
759
760
// If set to true, metric calculator will run
761
DECLARE_Bool(enable_metric_calculator);
762
763
// max consumer num in one data consumer group, for routine load
764
DECLARE_mInt32(max_consumer_num_per_group);
765
766
// the max size of thread pool for routine load task.
767
// this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5)
768
DECLARE_Int32(max_routine_load_thread_pool_size);
769
770
// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
771
// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
772
DECLARE_mInt32(max_memory_sink_batch_count);
773
774
// This configuration is used for the context gc thread schedule period
775
// note: unit is minute, default is 5min
776
DECLARE_mInt32(scan_context_gc_interval_min);
777
778
// es scroll keep-alive
779
DECLARE_String(es_scroll_keepalive);
780
781
// HTTP connection timeout for es
782
DECLARE_mInt32(es_http_timeout_ms);
783
784
// the max client cache number per each host
785
// There are variety of client cache in BE, but currently we use the
786
// same cache size configuration.
787
// TODO(cmy): use different config to set different client cache if necessary.
788
DECLARE_Int32(max_client_cache_size_per_host);
789
790
DECLARE_Int32(max_master_fe_client_cache_size);
791
792
// Dir to save files downloaded by SmallFileMgr
793
DECLARE_String(small_file_dir);
794
// path gc
795
DECLARE_Bool(path_gc_check);
796
DECLARE_mInt32(path_gc_check_interval_second);
797
DECLARE_mInt32(path_gc_check_step);
798
DECLARE_mInt32(path_gc_check_step_interval_ms);
799
800
// The following 2 configs limit the max usage of disk capacity of a data dir.
801
// If both of these 2 threshold reached, no more data can be writen into that data dir.
802
// The percent of max used capacity of a data dir
803
DECLARE_mInt32(storage_flood_stage_usage_percent); // 90%
804
// The min bytes that should be left of a data dir
805
DECLARE_mInt64(storage_flood_stage_left_capacity_bytes); // 1GB
806
// number of thread for flushing memtable per store
807
DECLARE_Int32(flush_thread_num_per_store);
808
// number of thread for flushing memtable per store, for high priority load task
809
DECLARE_Int32(high_priority_flush_thread_num_per_store);
810
// number of threads = min(flush_thread_num_per_store * num_store,
811
//                         max_flush_thread_num_per_cpu * num_cpu)
812
DECLARE_Int32(max_flush_thread_num_per_cpu);
813
814
// workload group flush pool params
815
DECLARE_mInt32(wg_flush_thread_num_per_store);
816
DECLARE_mInt32(wg_flush_thread_num_per_cpu);
817
818
// config for tablet meta checkpoint
819
DECLARE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num);
820
DECLARE_mInt32(tablet_meta_checkpoint_min_interval_secs);
821
DECLARE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs);
822
823
// config for default rowset type
824
// Valid configs: ALPHA, BETA
825
DECLARE_String(default_rowset_type);
826
827
// Maximum size of a single message body in all protocols
828
DECLARE_Int64(brpc_max_body_size);
829
// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
830
// Default, if the physical memory is less than or equal to 64G, the value is 1G
831
//          if the physical memory is greater than 64G, the value is physical memory * mem_limit(0.8) / 1024 * 20
832
DECLARE_Int64(brpc_socket_max_unwritten_bytes);
833
// TODO(zxy): expect to be true in v1.3
834
// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
835
// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
836
// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
837
DECLARE_mBool(transfer_large_data_by_brpc);
838
839
// max number of txns for every txn_partition_map in txn manager
840
// this is a self protection to avoid too many txns saving in manager
841
DECLARE_mInt64(max_runnings_transactions_per_txn_map);
842
843
// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
844
// this is a an enhancement for better performance to manage tablet
845
DECLARE_Int32(tablet_map_shard_size);
846
847
// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
848
// this is a an enhancement for better performance to manage txn
849
DECLARE_Int32(txn_map_shard_size);
850
851
// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
852
// this is a an enhancement for better performance to commit and publish txn
853
DECLARE_Int32(txn_shard_size);
854
855
// Whether to continue to start be when load tablet from header failed.
856
DECLARE_Bool(ignore_load_tablet_failure);
857
858
// Whether to continue to start be when load tablet from header failed.
859
DECLARE_mBool(ignore_rowset_stale_unconsistent_delete);
860
861
// Set max cache's size of query results, the unit is M byte
862
DECLARE_Int32(query_cache_max_size_mb);
863
864
// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
865
DECLARE_Int32(query_cache_elasticity_size_mb);
866
867
// Maximum number of cache partitions corresponding to a SQL
868
DECLARE_Int32(query_cache_max_partition_count);
869
870
// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
871
// the load process will reject new incoming load job of this tablet.
872
// This is to avoid too many version num.
873
DECLARE_mInt32(max_tablet_version_num);
874
875
DECLARE_mInt32(time_series_max_tablet_version_num);
876
877
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
878
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
879
DECLARE_String(thrift_server_type_of_fe);
880
881
// disable zone map index when page row is too few
882
DECLARE_mInt32(zone_map_row_num_threshold);
883
884
// aws sdk log level
885
//    Off = 0,
886
//    Fatal = 1,
887
//    Error = 2,
888
//    Warn = 3,
889
//    Info = 4,
890
//    Debug = 5,
891
//    Trace = 6
892
DECLARE_Int32(aws_log_level);
893
894
// azure sdk log level
895
//    Verbose = 1,
896
//    Informational = 2,
897
//    Warning = 3,
898
//    Error = 4
899
DECLARE_Int32(azure_log_level);
900
901
// the buffer size when read data from remote storage like s3
902
DECLARE_mInt32(remote_storage_read_buffer_mb);
903
904
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
905
// smaller than this value will continue to accumulate. specified as number of bytes.
906
// Decreasing this value will increase the frequency of consume/release.
907
// Increasing this value will cause MemTracker statistics to be inaccurate.
908
DECLARE_mInt32(mem_tracker_consume_min_size_bytes);
909
910
// The version information of the tablet will be stored in the memory
911
// in an adjacency graph data structure.
912
// And as the new version is written and the old version is deleted,
913
// the data structure will begin to have empty vertex with no edge associations(orphan vertex).
914
// This config is used to control that when the proportion of orphan vertex is greater than the threshold,
915
// the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely.
916
// This config usually only needs to be modified during testing.
917
// In most cases, it does not need to be modified.
918
DECLARE_mDouble(tablet_version_graph_orphan_vertex_ratio);
919
920
// share delta writers when memtable_on_sink_node = true
921
DECLARE_Bool(share_delta_writers);
922
// timeout for open load stream rpc in ms
923
DECLARE_Int64(open_load_stream_timeout_ms);
924
// enable write background when using brpc stream
925
DECLARE_mBool(enable_brpc_stream_write_background);
926
927
// brpc streaming max_buf_size in bytes
928
DECLARE_Int64(load_stream_max_buf_size);
929
// brpc streaming messages_in_batch
930
DECLARE_Int32(load_stream_messages_in_batch);
931
// brpc streaming StreamWait seconds on EAGAIN
932
DECLARE_Int32(load_stream_eagain_wait_seconds);
933
// max tasks per flush token in load stream
934
DECLARE_Int32(load_stream_flush_token_max_tasks);
935
// max wait flush token time in load stream
936
DECLARE_Int32(load_stream_max_wait_flush_token_time_ms);
937
// number of send batch thread pool size
938
DECLARE_Int32(send_batch_thread_pool_thread_num);
939
// number of send batch thread pool queue size
940
DECLARE_Int32(send_batch_thread_pool_queue_size);
941
942
// Limit the number of segment of a newly created rowset.
943
// The newly created rowset may to be compacted after loading,
944
// so if there are too many segment in a rowset, the compaction process
945
// will run out of memory.
946
// When doing compaction, each segment may take at least 1MB buffer.
947
DECLARE_mInt32(max_segment_num_per_rowset);
948
949
// Store segment without compression if a segment is smaller than
950
// segment_compression_threshold_kb.
951
DECLARE_mInt32(segment_compression_threshold_kb);
952
953
// The connection timeout when connecting to external table such as odbc table.
954
DECLARE_mInt32(external_table_connect_timeout_sec);
955
956
// Time to clean up useless JDBC connection pool cache
957
DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
958
959
// Global bitmap cache capacity for aggregation cache, size in bytes
960
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
961
DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
962
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
963
964
// A common object cache depends on an Sharded LRU Cache.
965
DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
966
967
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
968
// If the dependent kafka broker version older than 0.10.0.0,
969
// the value of kafka_api_version_request should be false, and the
970
// value set by the fallback version kafka_broker_version_fallback will be used,
971
// and the valid values are: 0.9.0.x, 0.8.x.y.
972
DECLARE_String(kafka_api_version_request);
973
DECLARE_String(kafka_broker_version_fallback);
974
DECLARE_mString(kafka_debug);
975
976
// The number of pool siz of routine load consumer.
977
// If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608
978
// Change this size to 0 to fix it temporarily.
979
DECLARE_mInt32(routine_load_consumer_pool_size);
980
981
// the timeout of condition variable wait in blocking_get and blocking_put
982
DECLARE_mInt32(blocking_queue_cv_wait_timeout_ms);
983
984
// Used in single-stream-multi-table load. When receive a batch of messages from kafka,
985
// if the size of batch is more than this threshold, we will request plans for all related tables.
986
DECLARE_Int32(multi_table_batch_plan_threshold);
987
988
// Used in single-stream-multi-table load. When receiving a batch of messages from Kafka,
989
// if the size of the table wait for plan is more than this threshold, we will request plans for all related tables.
990
// The param is aimed to avoid requesting and executing too many plans at once.
991
// Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC
992
// and improve the real-time processing of data.
993
DECLARE_Int32(multi_table_max_wait_tables);
994
995
// When the timeout of a load task is less than this threshold,
996
// Doris treats it as a high priority task.
997
// high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic.
998
// this threshold is mainly used to identify routine load tasks and should not be modified if not necessary.
999
DECLARE_mInt32(load_task_high_priority_threshold_second);
1000
1001
// The min timeout of load rpc (add batch, close)
1002
// Because a load rpc may be blocked for a while.
1003
// Increase this config may avoid rpc timeout.
1004
DECLARE_mInt32(min_load_rpc_timeout_ms);
1005
1006
// use which protocol to access function service, candicate is baidu_std/h2:grpc
1007
DECLARE_String(function_service_protocol);
1008
1009
// use which load balancer to select server to connect
1010
DECLARE_String(rpc_load_balancer);
1011
1012
// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
1013
// so we set a soft limit, default is 1MB
1014
DECLARE_mInt32(string_type_length_soft_limit_bytes);
1015
1016
DECLARE_mInt32(jsonb_type_length_soft_limit_bytes);
1017
1018
// Threshold fo reading a small file into memory
1019
DECLARE_mInt32(in_memory_file_size);
1020
1021
// ParquetReaderWrap prefetch buffer size
1022
DECLARE_Int32(parquet_reader_max_buffer_size);
1023
// Max size of parquet page header in bytes
1024
DECLARE_mInt32(parquet_header_max_size_mb);
1025
// Max buffer size for parquet row group
1026
DECLARE_mInt32(parquet_rowgroup_max_buffer_mb);
1027
// Max buffer size for parquet chunk column
1028
DECLARE_mInt32(parquet_column_max_buffer_mb);
1029
// Merge small IO, the max amplified read ratio
1030
DECLARE_mDouble(max_amplified_read_ratio);
1031
// Equivalent min size of each IO that can reach the maximum storage speed limit
1032
// 1MB for oss, 8KB for hdfs
1033
DECLARE_mInt32(merged_oss_min_io_size);
1034
DECLARE_mInt32(merged_hdfs_min_io_size);
1035
1036
// OrcReader
1037
DECLARE_mInt32(orc_natural_read_size_mb);
1038
DECLARE_mInt64(big_column_size_buffer);
1039
DECLARE_mInt64(small_column_size_buffer);
1040
1041
DECLARE_mInt32(runtime_filter_sampling_frequency);
1042
DECLARE_mInt32(execution_max_rpc_timeout_sec);
1043
DECLARE_mBool(execution_ignore_eovercrowded);
1044
1045
// cooldown task configs
1046
DECLARE_Int32(cooldown_thread_num);
1047
DECLARE_mInt64(generate_cooldown_task_interval_sec);
1048
DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h
1049
DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
1050
DECLARE_Int32(cold_data_compaction_thread_num);
1051
DECLARE_mInt32(cold_data_compaction_interval_sec);
1052
1053
DECLARE_Int32(min_s3_file_system_thread_num);
1054
DECLARE_Int32(max_s3_file_system_thread_num);
1055
1056
DECLARE_Bool(enable_time_lut);
1057
DECLARE_mBool(enable_simdjson_reader);
1058
1059
DECLARE_mBool(enable_query_like_bloom_filter);
1060
// number of s3 scanner thread pool size
1061
DECLARE_Int32(doris_remote_scanner_thread_pool_thread_num);
1062
// number of s3 scanner thread pool queue size
1063
DECLARE_Int32(doris_remote_scanner_thread_pool_queue_size);
1064
1065
// limit the queue of pending batches which will be sent by a single nodechannel
1066
DECLARE_mInt64(nodechannel_pending_queue_max_bytes);
1067
1068
// The batch size for sending data by brpc streaming client
1069
DECLARE_mInt64(brpc_streaming_client_batch_bytes);
1070
DECLARE_mInt64(block_cache_wait_timeout_ms);
1071
1072
DECLARE_Bool(enable_brpc_builtin_services);
1073
1074
// Max waiting time to wait the "plan fragment start" rpc.
1075
// If timeout, the fragment will be cancelled.
1076
// This parameter is usually only used when the FE loses connection,
1077
// and the BE can automatically cancel the relevant fragment after the timeout,
1078
// so as to avoid occupying the execution thread for a long time.
1079
DECLARE_mInt32(max_fragment_start_wait_time_seconds);
1080
1081
// Node role tag for backend. Mix role is the default role, and computation role have no
1082
// any tablet.
1083
DECLARE_String(be_node_role);
1084
1085
// Hide webserver page for safety.
1086
// Hide the be config page for webserver.
1087
DECLARE_Bool(hide_webserver_config_page);
1088
1089
DECLARE_Bool(enable_segcompaction);
1090
1091
// Max number of segments allowed in a single segcompaction task.
1092
DECLARE_Int32(segcompaction_batch_size);
1093
1094
// Max row count allowed in a single source segment, bigger segments will be skipped.
1095
DECLARE_Int32(segcompaction_candidate_max_rows);
1096
1097
// Max file size allowed in a single source segment, bigger segments will be skipped.
1098
DECLARE_Int64(segcompaction_candidate_max_bytes);
1099
1100
// Max total row count allowed in a single segcompaction task.
1101
DECLARE_Int32(segcompaction_task_max_rows);
1102
1103
// Max total file size allowed in a single segcompaction task.
1104
DECLARE_Int64(segcompaction_task_max_bytes);
1105
1106
// Global segcompaction thread pool size.
1107
DECLARE_mInt32(segcompaction_num_threads);
1108
1109
// enable java udf and jdbc scannode
1110
DECLARE_Bool(enable_java_support);
1111
1112
// Set config randomly to check more issues in github workflow
1113
DECLARE_Bool(enable_fuzzy_mode);
1114
1115
DECLARE_Bool(enable_debug_points);
1116
1117
DECLARE_Int32(pipeline_executor_size);
1118
1119
// block file cache
1120
DECLARE_Bool(enable_file_cache);
1121
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
1122
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
1123
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}]
1124
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
1125
// Note1: storage is "disk" by default
1126
// Note2: when the storage is "memory", the path is ignored. So you can set xxx to anything you like
1127
// and doris will just reset the path to "memory" internally.
1128
// In a very wierd case when your storage is disk, and the directory, by accident, is named
1129
// "memory" for some reason, you should write the path as:
1130
//     {"path": "memory", "total_size":53687091200, "storage": "disk"}
1131
// or use the default storage value:
1132
//     {"path": "memory", "total_size":53687091200}
1133
// Both will use the directory "memory" on the disk instead of the real RAM.
1134
DECLARE_String(file_cache_path);
1135
DECLARE_Int64(file_cache_each_block_size);
1136
DECLARE_Bool(clear_file_cache);
1137
DECLARE_Bool(enable_file_cache_query_limit);
1138
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
1139
DECLARE_Int32(file_cache_exit_disk_resource_limit_mode_percent);
1140
DECLARE_mBool(enable_evict_file_cache_in_advance);
1141
DECLARE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent);
1142
DECLARE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent);
1143
DECLARE_mInt32(file_cache_evict_in_advance_interval_ms);
1144
DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes);
1145
DECLARE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold);
1146
DECLARE_mBool(enable_read_cache_file_directly);
1147
DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size);
1148
// If true, evict the ttl cache using LRU when full.
1149
// Otherwise, only expiration can evict ttl and new data won't add to cache when full.
1150
DECLARE_Bool(enable_ttl_cache_evict_using_lru);
1151
DECLARE_mBool(enbale_dump_error_file);
1152
// limit the max size of error log on disk
1153
DECLARE_mInt64(file_cache_error_log_limit_bytes);
1154
DECLARE_mInt64(cache_lock_wait_long_tail_threshold_us);
1155
DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
1156
1157
// Base compaction may retrieve and produce some less frequently accessed data,
1158
// potentially affecting the file cache hit rate.
1159
// This configuration determines whether to retain the output within the file cache.
1160
// Make your choice based on the following considerations:
1161
// If your file cache is ample enough to accommodate all the data in your database,
1162
// enable this option; otherwise, it is recommended to leave it disabled.
1163
DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
1164
DECLARE_mInt64(file_cache_remove_block_qps_limit);
1165
DECLARE_mInt64(file_cache_background_gc_interval_ms);
1166
DECLARE_mBool(enable_reader_dryrun_when_download_file_cache);
1167
DECLARE_mInt64(file_cache_background_monitor_interval_ms);
1168
DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms);
1169
DECLARE_mInt64(file_cache_background_ttl_gc_batch);
1170
DECLARE_Int32(file_cache_downloader_thread_num_min);
1171
DECLARE_Int32(file_cache_downloader_thread_num_max);
1172
// used to persist lru information before be reboot and load the info back
1173
DECLARE_mInt64(file_cache_background_lru_dump_interval_ms);
1174
// dump queue only if the queue update specific times through several dump intervals
1175
DECLARE_mInt64(file_cache_background_lru_dump_update_cnt_threshold);
1176
DECLARE_mInt64(file_cache_background_lru_dump_tail_record_num);
1177
DECLARE_mInt64(file_cache_background_lru_log_replay_interval_ms);
1178
DECLARE_mBool(enable_evaluate_shadow_queue_diff);
1179
1180
// inverted index searcher cache
1181
// cache entry stay time after lookup
1182
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
1183
// cache entry that have not been visited for a certain period of time can be cleaned up by GC thread
1184
DECLARE_mInt32(inverted_index_cache_stale_sweep_time_sec);
1185
// inverted index searcher cache size
1186
DECLARE_String(inverted_index_searcher_cache_limit);
1187
// set `true` to enable insert searcher into cache when write inverted index data
1188
DECLARE_mBool(enable_write_index_searcher_cache);
1189
DECLARE_Bool(enable_inverted_index_cache_check_timestamp);
1190
DECLARE_mBool(enable_inverted_index_correct_term_write);
1191
DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50%
1192
DECLARE_Int32(inverted_index_query_cache_shards);
1193
1194
// inverted index match bitmap cache size
1195
DECLARE_String(inverted_index_query_cache_limit);
1196
1197
// inverted index
1198
DECLARE_mDouble(inverted_index_ram_buffer_size);
1199
DECLARE_mInt32(inverted_index_max_buffered_docs);
1200
// dict path for chinese analyzer
1201
DECLARE_String(inverted_index_dict_path);
1202
DECLARE_Int32(inverted_index_read_buffer_size);
1203
// tree depth for bkd index
1204
DECLARE_Int32(max_depth_in_bkd_tree);
1205
// index compaction
1206
DECLARE_mBool(inverted_index_compaction_enable);
1207
// Only for debug, do not use in production
1208
DECLARE_mBool(debug_inverted_index_compaction);
1209
// index by RAM directory
1210
DECLARE_mBool(inverted_index_ram_dir_enable);
1211
// wheather index by RAM directory when base compaction
1212
DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction);
1213
// use num_broadcast_buffer blocks as buffer to do broadcast
1214
DECLARE_Int32(num_broadcast_buffer);
1215
1216
// max depth of expression tree allowed.
1217
DECLARE_Int32(max_depth_of_expr_tree);
1218
1219
// Report a tablet as bad when io errors occurs more than this value.
1220
DECLARE_mInt64(max_tablet_io_errors);
1221
1222
// Report a tablet as bad when its path not found
1223
DECLARE_Int32(tablet_path_check_interval_seconds);
1224
DECLARE_mInt32(tablet_path_check_batch_size);
1225
1226
// it must be larger than or equal to 5MB
1227
DECLARE_mInt64(s3_write_buffer_size);
1228
// Log interval when doing s3 upload task
1229
DECLARE_mInt32(s3_file_writer_log_interval_second);
1230
// the max number of cached file handle for block segemnt
1231
DECLARE_mInt64(file_cache_max_file_reader_cache_size);
1232
DECLARE_mInt64(hdfs_write_batch_buffer_size_mb);
1233
//enable shrink memory
1234
DECLARE_mBool(enable_shrink_memory);
1235
// enable cache for high concurrent point query work load
1236
DECLARE_mInt32(schema_cache_capacity);
1237
DECLARE_mInt32(schema_cache_sweep_time_sec);
1238
1239
// max number of segment cache
1240
DECLARE_Int32(segment_cache_capacity);
1241
DECLARE_Int32(segment_cache_fd_percentage);
1242
DECLARE_Int32(segment_cache_memory_percentage);
1243
DECLARE_Bool(enable_segment_cache_prune);
1244
1245
DECLARE_mInt32(estimated_mem_per_column_reader);
1246
1247
// enable binlog
1248
DECLARE_Bool(enable_feature_binlog);
1249
1250
// enable set in BitmapValue
1251
DECLARE_Bool(enable_set_in_bitmap_value);
1252
1253
// max number of hdfs file handle in cache
1254
DECLARE_Int64(max_hdfs_file_handle_cache_num);
1255
DECLARE_Int32(max_hdfs_file_handle_cache_time_sec);
1256
1257
// max number of meta info of external files, such as parquet footer
1258
DECLARE_Int64(max_external_file_meta_cache_num);
1259
// Apply delete pred in cumu compaction
1260
DECLARE_mBool(enable_delete_when_cumu_compaction);
1261
1262
// max_write_buffer_number for rocksdb
1263
DECLARE_Int32(rocksdb_max_write_buffer_number);
1264
1265
// Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false.
1266
DECLARE_mBool(allow_zero_date);
1267
// Allow invalid decimalv2 literal for compatible with old version. Recommend set it false strongly.
1268
DECLARE_mBool(allow_invalid_decimalv2_literal);
1269
// Allow to specify kerberos credentials cache path.
1270
DECLARE_mString(kerberos_ccache_path);
1271
// set krb5.conf path, use "/etc/krb5.conf" by default
1272
DECLARE_mString(kerberos_krb5_conf_path);
1273
// the interval for renew kerberos ticket cache
1274
DECLARE_mInt32(kerberos_refresh_interval_second);
1275
1276
// JDK-8153057: avoid StackOverflowError thrown from the UncaughtExceptionHandler in thread "process reaper"
1277
DECLARE_mBool(jdk_process_reaper_use_default_stack_size);
1278
1279
// Values include `none`, `glog`, `boost`, `glibc`, `libunwind`
1280
DECLARE_mString(get_stack_trace_tool);
1281
DECLARE_mBool(enable_address_sanitizers_with_stack_trace);
1282
1283
// DISABLED: Don't resolve location info.
1284
// FAST: Perform CU lookup using .debug_aranges (might be incomplete).
1285
// FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure.
1286
// FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL.
1287
DECLARE_mString(dwarf_location_info_mode);
1288
1289
// the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
1290
DECLARE_mInt64(auto_inc_prefetch_size_ratio);
1291
1292
// the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer
1293
DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
1294
1295
// number of threads that fetch auto-inc ranges from FE
1296
DECLARE_mInt64(auto_inc_fetch_thread_num);
1297
// Max connection cache num for point lookup queries
1298
DECLARE_mInt64(lookup_connection_cache_capacity);
1299
1300
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
1301
DECLARE_mInt64(LZ4_HC_compression_level);
1302
// Threshold of a column as sparse column
1303
// Notice: TEST ONLY
1304
DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column);
1305
DECLARE_mBool(variant_use_cloud_schema_dict_cache);
1306
// Threshold to estimate a column is sparsed
1307
// Treat invalid json format str as string, instead of throwing exception if false
1308
DECLARE_mBool(variant_throw_exeception_on_invalid_json);
1309
1310
DECLARE_mBool(enable_merge_on_write_correctness_check);
1311
// USED FOR DEBUGING
1312
// core directly if the compaction found there's duplicate key on mow table
1313
DECLARE_mBool(enable_mow_compaction_correctness_check_core);
1314
// rowid conversion correctness check when compaction for mow table
1315
DECLARE_mBool(enable_rowid_conversion_correctness_check);
1316
// missing rows correctness check when compaction for mow table
1317
DECLARE_mBool(enable_missing_rows_correctness_check);
1318
// When the number of missing versions is more than this value, do not directly
1319
// retry the publish and handle it through async publish.
1320
DECLARE_mInt32(mow_publish_max_discontinuous_version_num);
1321
// When the version is not continuous for MOW table in publish phase and the gap between
1322
// current txn's publishing version and the max version of the tablet exceeds this value,
1323
// don't print warning log
1324
DECLARE_mInt32(publish_version_gap_logging_threshold);
1325
// get agg by cache for mow table
1326
DECLARE_mBool(enable_mow_get_agg_by_cache);
1327
// get agg correctness check for mow table
1328
DECLARE_mBool(enable_mow_get_agg_correctness_check_core);
1329
DECLARE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap);
1330
DECLARE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap);
1331
1332
// The secure path with user files, used in the `local` table function.
1333
DECLARE_mString(user_files_secure_path);
1334
1335
// If fe's frontend info has not been updated for more than fe_expire_duration_seconds, it will be regarded
1336
// as an abnormal fe, this will cause be to cancel this fe's related query.
1337
DECLARE_Int32(fe_expire_duration_seconds);
1338
1339
// If use stop_be.sh --grace, then BE has to wait all running queries to stop to avoiding running query failure
1340
// , but if the waiting time exceed the limit, then be will exit directly.
1341
// During this period, FE will not send any queries to BE and waiting for all running queries to stop.
1342
DECLARE_Int32(grace_shutdown_wait_seconds);
1343
1344
// BitmapValue serialize version.
1345
DECLARE_Int16(bitmap_serialize_version);
1346
1347
// group commit config
1348
DECLARE_String(group_commit_wal_path);
1349
DECLARE_Int32(group_commit_replay_wal_retry_num);
1350
DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds);
1351
DECLARE_Int32(group_commit_replay_wal_retry_interval_max_seconds);
1352
DECLARE_mInt32(group_commit_relay_wal_threads);
1353
// This config can be set to limit thread number in group commit request fragment thread pool.
1354
DECLARE_mInt32(group_commit_insert_threads);
1355
DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
1356
DECLARE_Bool(wait_internal_group_commit_finish);
1357
// Max size(bytes) of group commit queues, used for mem back pressure.
1358
DECLARE_mInt32(group_commit_queue_mem_limit);
1359
// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
1360
// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
1361
DECLARE_mString(group_commit_wal_max_disk_limit);
1362
DECLARE_Bool(group_commit_wait_replay_wal_finish);
1363
1364
// The configuration item is used to lower the priority of the scanner thread,
1365
// typically employed to ensure CPU scheduling for write operations.
1366
// Default is 0, which is default value of thread nice value, increase this value
1367
// to lower the priority of scan threads
1368
DECLARE_Int32(scan_thread_nice_value);
1369
// Used to modify the recycle interval of tablet schema cache
1370
DECLARE_mInt32(tablet_schema_cache_recycle_interval);
1371
// Granularity is at the column level
1372
DECLARE_mInt32(tablet_schema_cache_capacity);
1373
1374
// Use `LOG(FATAL)` to replace `throw` when true
1375
DECLARE_mBool(exit_on_exception);
1376
1377
// cgroup
1378
DECLARE_String(doris_cgroup_cpu_path);
1379
DECLARE_mBool(enable_be_proc_monitor);
1380
DECLARE_mInt32(be_proc_monitor_interval_ms);
1381
DECLARE_Int32(workload_group_metrics_interval_ms);
1382
1383
DECLARE_mBool(enable_workload_group_memory_gc);
1384
1385
// This config controls whether the s3 file writer would flush cache asynchronously
1386
DECLARE_Bool(enable_flush_file_cache_async);
1387
1388
// Remove predicate that is always true for a segment.
1389
DECLARE_Bool(ignore_always_true_predicate_for_segment);
1390
1391
// Ingest binlog work pool size
1392
DECLARE_Int32(ingest_binlog_work_pool_size);
1393
1394
// Ingest binlog with persistent connection
1395
DECLARE_Bool(enable_ingest_binlog_with_persistent_connection);
1396
1397
// Download binlog rate limit, unit is KB/s
1398
DECLARE_Int32(download_binlog_rate_limit_kbs);
1399
1400
DECLARE_mInt32(buffered_reader_read_timeout_ms);
1401
1402
// whether to enable /api/snapshot api
1403
DECLARE_Bool(enable_snapshot_action);
1404
1405
// The max columns size for a tablet schema
1406
DECLARE_mInt32(variant_max_merged_tablet_schema_size);
1407
1408
// The max sparse column statistics size for a variant column
1409
DECLARE_mInt32(variant_max_sparse_column_statistics_size);
1410
1411
DECLARE_mInt64(local_exchange_buffer_mem_limit);
1412
1413
DECLARE_mInt64(enable_debug_log_timeout_secs);
1414
1415
DECLARE_mBool(enable_column_type_check);
1416
1417
// Tolerance for the number of partition id 0 in rowset, default 0
1418
DECLARE_Int32(ignore_invalid_partition_id_rowset_num);
1419
1420
DECLARE_mInt32(report_query_statistics_interval_ms);
1421
DECLARE_mInt32(query_statistics_reserve_timeout_ms);
1422
DECLARE_mInt32(report_exec_status_thread_num);
1423
1424
// consider two high usage disk at the same available level if they do not exceed this diff.
1425
DECLARE_mDouble(high_disk_avail_level_diff_usages);
1426
1427
// create tablet in partition random robin idx lru size, default 10000
1428
DECLARE_Int32(partition_disk_index_lru_size);
1429
DECLARE_String(spill_storage_root_path);
1430
// Spill storage limit specified as number of bytes
1431
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
1432
// or percentage of capaity ('<int>%').
1433
// Defaults to bytes if no unit is given.
1434
// Must larger than 0.
1435
// If specified as percentage, the final limit value is:
1436
//   disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit
1437
DECLARE_String(spill_storage_limit);
1438
DECLARE_mInt32(spill_gc_interval_ms);
1439
DECLARE_mInt32(spill_gc_work_time_ms);
1440
DECLARE_Int32(spill_io_thread_pool_thread_num);
1441
DECLARE_Int32(spill_io_thread_pool_queue_size);
1442
1443
DECLARE_mBool(check_segment_when_build_rowset_meta);
1444
1445
DECLARE_Int32(num_query_ctx_map_partitions);
1446
1447
DECLARE_mBool(force_azure_blob_global_endpoint);
1448
1449
DECLARE_mBool(enable_s3_rate_limiter);
1450
DECLARE_mInt64(s3_get_bucket_tokens);
1451
DECLARE_mInt64(s3_get_token_per_second);
1452
DECLARE_mInt64(s3_get_token_limit);
1453
1454
DECLARE_mInt64(s3_put_bucket_tokens);
1455
DECLARE_mInt64(s3_put_token_per_second);
1456
DECLARE_mInt64(s3_put_token_limit);
1457
// max s3 client retry times
1458
DECLARE_mInt32(max_s3_client_retry);
1459
// When meet s3 429 error, the "get" request will
1460
// sleep s3_read_base_wait_time_ms (*1, *2, *3, *4) ms
1461
// get try again.
1462
// The max sleep time is s3_read_max_wait_time_ms
1463
// and the max retry time is max_s3_client_retry
1464
DECLARE_mInt32(s3_read_base_wait_time_ms);
1465
DECLARE_mInt32(s3_read_max_wait_time_ms);
1466
DECLARE_mBool(enable_s3_object_check_after_upload);
1467
1468
// write as inverted index tmp directory
1469
DECLARE_String(tmp_file_dir);
1470
1471
// the directory for storing the trino-connector plugins.
1472
DECLARE_String(trino_connector_plugin_dir);
1473
1474
// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client
1475
DECLARE_mString(ca_cert_file_paths);
1476
1477
/** Table sink configurations(currently contains only external table types) **/
1478
// Minimum data processed to scale writers in exchange when non partition writing
1479
DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold);
1480
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1481
DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold);
1482
// Minimum partition data processed to rebalance writers in exchange when partition writing
1483
DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold);
1484
// Maximum processed partition nums of per writer when partition writing
1485
DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer);
1486
1487
/** Hive sink configurations **/
1488
DECLARE_mInt64(hive_sink_max_file_size);
1489
1490
/** Iceberg sink configurations **/
1491
DECLARE_mInt64(iceberg_sink_max_file_size);
1492
1493
// Number of open tries, default 1 means only try to open once.
1494
// Retry the Open num_retries time waiting 100 milliseconds between retries.
1495
DECLARE_mInt32(thrift_client_open_num_tries);
1496
1497
// http scheme in S3Client to use. E.g. http or https
1498
DECLARE_String(s3_client_http_scheme);
1499
1500
DECLARE_mBool(ignore_schema_change_check);
1501
1502
/** Only use in fuzzy test **/
1503
DECLARE_mInt64(string_overflow_size);
1504
1505
// The min thread num for BufferedReaderPrefetchThreadPool
1506
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread);
1507
// The max thread num for BufferedReaderPrefetchThreadPool
1508
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread);
1509
// The min thread num for S3FileUploadThreadPool
1510
DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread);
1511
// The max thread num for S3FileUploadThreadPool
1512
DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread);
1513
// The maximum jvm heap usage ratio for hdfs write workload
1514
DECLARE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio);
1515
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
1516
DECLARE_mInt64(hdfs_jni_write_sleep_milliseconds);
1517
// The max retry times when hdfs write failed
1518
DECLARE_mInt64(hdfs_jni_write_max_retry_time);
1519
1520
// The min thread num for NonBlockCloseThreadPool
1521
DECLARE_Int64(min_nonblock_close_thread_num);
1522
// The max thread num for NonBlockCloseThreadPool
1523
DECLARE_Int64(max_nonblock_close_thread_num);
1524
// The possibility that mem allocator throws an exception during memory allocation
1525
// This config is for test usage, be careful when changing it.
1526
DECLARE_mDouble(mem_alloc_fault_probability);
1527
// The time out milliseconds for remote fetch schema RPC
1528
DECLARE_mInt64(fetch_remote_schema_rpc_timeout_ms);
1529
// The size of the local buffer for S3FileSytem's upload function
1530
1531
DECLARE_Int64(s3_file_system_local_upload_buffer_size);
1532
1533
//JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues.
1534
DECLARE_Bool(enable_jvm_monitor);
1535
1536
// Num threads to load data dirs, default value -1 indicates the same number of threads as the number of data dirs
1537
DECLARE_Int32(load_data_dirs_threads);
1538
1539
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
1540
DECLARE_mBool(skip_loading_stale_rowset_meta);
1541
// Whether to use file to record log. When starting BE with --console,
1542
// all logs will be written to both standard output and file.
1543
// Disable this option will no longer use file to record log.
1544
// Only works when starting BE with --console.
1545
DECLARE_Bool(enable_file_logger);
1546
1547
// The minimum row group size when exporting Parquet files.
1548
DECLARE_Int64(min_row_group_size);
1549
1550
DECLARE_mInt64(compaction_memory_bytes_limit);
1551
1552
DECLARE_mInt64(compaction_batch_size);
1553
1554
DECLARE_mBool(enable_parquet_page_index);
1555
1556
// Wheather to ignore not found file in external teble(eg, hive)
1557
// Default is true, if set to false, the not found file will result in query failure.
1558
DECLARE_mBool(ignore_not_found_file_in_external_table);
1559
1560
DECLARE_mBool(enable_hdfs_mem_limiter);
1561
1562
// Define how many percent data in hashtable bigger than limit
1563
// we should do agg limit opt
1564
DECLARE_mInt16(topn_agg_limit_multiplier);
1565
1566
DECLARE_mInt64(tablet_meta_serialize_size_limit);
1567
1568
DECLARE_mInt64(pipeline_task_leakage_detect_period_secs);
1569
// To be compatible with hadoop's block compression
1570
DECLARE_mInt32(snappy_compression_block_size);
1571
DECLARE_mInt32(lz4_compression_block_size);
1572
1573
DECLARE_mBool(enable_pipeline_task_leakage_detect);
1574
1575
// MB
1576
DECLARE_Int32(query_cache_size);
1577
DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
1578
1579
// Enable validation to check the correctness of table size.
1580
DECLARE_Bool(enable_table_size_correctness_check);
1581
// Enable sleep 5s between delete cumulative compaction.
1582
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
1583
1584
DECLARE_mInt32(compaction_num_per_round);
1585
1586
DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds);
1587
DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
1588
DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
1589
DECLARE_mInt32(schema_dict_cache_capacity);
1590
1591
DECLARE_mBool(enable_mow_verbose_log);
1592
1593
// whether to prune rows with delete sign = 1 in base compaction
1594
// ATTN: this config is only for test
1595
DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
1596
1597
// Because the root_path for the HDFS resource was previously passed an empty string (since v2.1),
1598
// which was incorrect, this configuration has been added to ensure compatibility
1599
// and guarantee that the root_path works as expected.
1600
DECLARE_Bool(enable_root_path_of_hdfs_resource);
1601
1602
DECLARE_mInt32(tablet_sched_delay_time_ms);
1603
DECLARE_mInt32(load_trigger_compaction_version_percent);
1604
DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
1605
DECLARE_mBool(enable_compaction_pause_on_high_memory);
1606
1607
DECLARE_mBool(enable_quorum_success_write);
1608
DECLARE_mDouble(quorum_success_max_wait_multiplier);
1609
DECLARE_mInt64(quorum_success_min_wait_seconds);
1610
DECLARE_mInt32(quorum_success_remaining_timeout_seconds);
1611
1612
DECLARE_mBool(enable_calc_delete_bitmap_between_segments_concurrently);
1613
1614
DECLARE_mBool(enable_fetch_rowsets_from_peer_replicas);
1615
// the max length of segments key bounds, in bytes
1616
// ATTENTION: as long as this conf has ever been enabled, cluster downgrade and backup recovery will no longer be supported.
1617
DECLARE_mInt32(segments_key_bounds_truncation_threshold);
1618
// ATTENTION: for test only, use random segments key bounds truncation threshold every time
1619
DECLARE_mBool(random_segments_key_bounds_truncation);
1620
1621
DECLARE_mBool(enable_auto_clone_on_compaction_missing_version);
1622
1623
DECLARE_mBool(enable_auto_clone_on_mow_publish_missing_version);
1624
1625
// The capacity of segment partial column cache, used to cache column readers for each segment.
1626
DECLARE_mInt32(max_segment_partial_column_cache_size);
1627
1628
#ifdef BE_TEST
1629
// test s3
1630
DECLARE_String(test_s3_resource);
1631
DECLARE_String(test_s3_ak);
1632
DECLARE_String(test_s3_sk);
1633
DECLARE_String(test_s3_endpoint);
1634
DECLARE_String(test_s3_region);
1635
DECLARE_String(test_s3_bucket);
1636
DECLARE_String(test_s3_prefix);
1637
#endif
1638
1639
class Register {
1640
public:
1641
    struct Field {
1642
        const char* type = nullptr;
1643
        const char* name = nullptr;
1644
        void* storage = nullptr;
1645
        const char* defval = nullptr;
1646
        bool valmutable = false;
1647
        Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1648
              bool fvalmutable)
1649
                : type(ftype),
1650
                  name(fname),
1651
                  storage(fstorage),
1652
                  defval(fdefval),
1653
709
                  valmutable(fvalmutable) {}
1654
    };
1655
1656
public:
1657
    static std::map<std::string, Field>* _s_field_map;
1658
1659
public:
1660
    Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1661
705
             bool fvalmutable) {
1662
705
        if (_s_field_map == nullptr) {
  Branch (1662:13): [True: 1, False: 704]
1663
1
            _s_field_map = new std::map<std::string, Field>();
1664
1
        }
1665
705
        Field field(ftype, fname, fstorage, fdefval, fvalmutable);
1666
705
        _s_field_map->insert(std::make_pair(std::string(fname), field));
1667
705
    }
1668
};
1669
1670
// RegisterConfValidator class is used to store validator function of registered config fields in
1671
// Register::_s_field_map.
1672
// If any validator return false when BE bootstart, the bootstart will be terminated.
1673
// If validator return false when use http API to update some config, the config will not
1674
// be modified and the API will return failure.
1675
class RegisterConfValidator {
1676
public:
1677
    // Validator for each config name.
1678
    static std::map<std::string, std::function<bool()>>* _s_field_validator;
1679
1680
public:
1681
    RegisterConfValidator(const char* fname, const std::function<bool()>& validator) {
1682
        if (_s_field_validator == nullptr) {
1683
            _s_field_validator = new std::map<std::string, std::function<bool()>>();
1684
        }
1685
        // register validator to _s_field_validator
1686
        _s_field_validator->insert(std::make_pair(std::string(fname), validator));
1687
    }
1688
};
1689
1690
// configuration properties load from config file.
1691
class Properties {
1692
public:
1693
    // load conf from file, if must_exist is true and file does not exist, return false
1694
    bool load(const char* conf_file, bool must_exist = true);
1695
1696
    // Find the config value by key from `file_conf_map`.
1697
    // If found, set `retval` to the config value,
1698
    // or set `retval` to `defstr`
1699
    // if retval is not set(in case defstr is nullptr), set is_retval_set to false
1700
    template <typename T>
1701
    bool get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set,
1702
                        std::string& rawval) const;
1703
1704
    void set(const std::string& key, const std::string& val);
1705
1706
    void set_force(const std::string& key, const std::string& val);
1707
1708
    // dump props to conf file
1709
    Status dump(const std::string& conffile);
1710
1711
private:
1712
    std::map<std::string, std::string> file_conf_map;
1713
};
1714
1715
// full configurations.
1716
extern std::map<std::string, std::string>* full_conf_map;
1717
1718
extern std::mutex custom_conf_lock;
1719
1720
// Init the config from `conf_file`.
1721
// If fill_conf_map is true, the updated config will also update the `full_conf_map`.
1722
// If must_exist is true and `conf_file` does not exist, this function will return false.
1723
// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
1724
bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist = true,
1725
          bool set_to_default = true);
1726
1727
Status set_config(const std::string& field, const std::string& value, bool need_persist = false,
1728
                  bool force = false);
1729
1730
Status persist_config(const std::string& field, const std::string& value);
1731
1732
std::mutex* get_mutable_string_config_lock();
1733
1734
std::vector<std::vector<std::string>> get_config_info();
1735
1736
Status set_fuzzy_configs();
1737
1738
void update_config(const std::string& field, const std::string& value);
1739
1740
} // namespace config
1741
} // namespace doris