Coverage Report

Created: 2024-11-20 15:53

/root/doris/be/src/common/config.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <cstdint>
21
#include <functional>
22
#include <map>
23
#include <mutex>
24
#include <string>
25
#include <utility>
26
#include <vector>
27
28
#define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME
29
30
#define DECLARE_Bool(name) DECLARE_FIELD(bool, name)
31
#define DECLARE_Int16(name) DECLARE_FIELD(int16_t, name)
32
#define DECLARE_Int32(name) DECLARE_FIELD(int32_t, name)
33
#define DECLARE_Int64(name) DECLARE_FIELD(int64_t, name)
34
#define DECLARE_Double(name) DECLARE_FIELD(double, name)
35
#define DECLARE_String(name) DECLARE_FIELD(std::string, name)
36
#define DECLARE_Bools(name) DECLARE_FIELD(std::vector<bool>, name)
37
#define DECLARE_Int16s(name) DECLARE_FIELD(std::vector<int16_t>, name)
38
#define DECLARE_Int32s(name) DECLARE_FIELD(std::vector<int32_t>, name)
39
#define DECLARE_Int64s(name) DECLARE_FIELD(std::vector<int64_t>, name)
40
#define DECLARE_Doubles(name) DECLARE_FIELD(std::vector<double>, name)
41
#define DECLARE_Strings(name) DECLARE_FIELD(std::vector<std::string>, name)
42
#define DECLARE_mBool(name) DECLARE_FIELD(bool, name)
43
#define DECLARE_mInt16(name) DECLARE_FIELD(int16_t, name)
44
#define DECLARE_mInt32(name) DECLARE_FIELD(int32_t, name)
45
#define DECLARE_mInt64(name) DECLARE_FIELD(int64_t, name)
46
#define DECLARE_mDouble(name) DECLARE_FIELD(double, name)
47
#define DECLARE_mString(name) DECLARE_FIELD(std::string, name)
48
49
#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE)                      \
50
21
    FIELD_TYPE FIELD_NAME;                                                                   \
51
21
    static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &(FIELD_NAME), FIELD_DEFAULT, \
52
21
                                     VALMUTABLE);
53
54
#define DEFINE_VALIDATOR(FIELD_NAME, VALIDATOR)              \
55
    static auto validator_##FIELD_NAME = VALIDATOR;          \
56
    static RegisterConfValidator reg_validator_##FIELD_NAME( \
57
            #FIELD_NAME, []() -> bool { return validator_##FIELD_NAME(FIELD_NAME); });
58
59
1
#define DEFINE_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false)
60
1
#define DEFINE_Bools(name, defaultstr) DEFINE_FIELD(std::vector<bool>, name, defaultstr, false)
61
1
#define DEFINE_Doubles(name, defaultstr) DEFINE_FIELD(std::vector<double>, name, defaultstr, false)
62
1
#define DEFINE_Int16s(name, defaultstr) DEFINE_FIELD(std::vector<int16_t>, name, defaultstr, false)
63
1
#define DEFINE_Int32s(name, defaultstr) DEFINE_FIELD(std::vector<int32_t>, name, defaultstr, false)
64
1
#define DEFINE_Int64s(name, defaultstr) DEFINE_FIELD(std::vector<int64_t>, name, defaultstr, false)
65
3
#define DEFINE_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false)
66
1
#define DEFINE_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false)
67
1
#define DEFINE_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false)
68
1
#define DEFINE_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false)
69
2
#define DEFINE_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false)
70
#define DEFINE_Strings(name, defaultstr) \
71
1
    DEFINE_FIELD(std::vector<std::string>, name, defaultstr, false)
72
1
#define DEFINE_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true)
73
1
#define DEFINE_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true)
74
1
#define DEFINE_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true)
75
1
#define DEFINE_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true)
76
1
#define DEFINE_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true)
77
1
#define DEFINE_mString(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, true)
78
#define DEFINE_Validator(name, validator) DEFINE_VALIDATOR(name, validator)
79
80
namespace doris {
81
class Status;
82
83
// If you want to modify the value of config, please go to common/config.cpp to modify.
84
namespace config {
85
// Dir of custom config file
86
DECLARE_String(custom_config_dir);
87
88
// Dir of jdbc drivers
89
DECLARE_String(jdbc_drivers_dir);
90
91
// cluster id
92
DECLARE_Int32(cluster_id);
93
// port on which BackendService is exported
94
DECLARE_Int32(be_port);
95
96
// port for brpc
97
DECLARE_Int32(brpc_port);
98
99
// port for arrow flight sql
100
// Default -1, do not start arrow flight sql server.
101
DECLARE_Int32(arrow_flight_sql_port);
102
103
// If the external client cannot directly access priority_networks, set public_host to be accessible
104
// to external client.
105
// There are usually two usage scenarios:
106
// 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network.
107
// However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be
108
// randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx.
109
// 2. if priority_networks is an internal network IP, and BE node has its own independent external IP,
110
// but Doris currently does not support modifying priority_networks, setting public_host to the real external IP.
111
DECLARE_mString(public_host);
112
113
// If the BE node is connected to the external network through a reverse proxy like Nginx
114
// and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy
115
// `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example:
116
// upstream arrowflight {
117
//    server 10.16.10.8:8069;
118
//    server 10.16.10.8:8068;
119
//}
120
// server {
121
//    listen 8167 http2;
122
//    listen [::]:8167 http2;
123
//    server_name doris.arrowflight.com;
124
// }
125
DECLARE_Int32(arrow_flight_sql_proxy_port);
126
127
// the number of bthreads for brpc, the default value is set to -1,
128
// which means the number of bthreads is #cpu-cores
129
DECLARE_Int32(brpc_num_threads);
130
DECLARE_Int32(brpc_idle_timeout_sec);
131
132
// Declare a selection strategy for those servers have many ips.
133
// Note that there should at most one ip match this list.
134
// This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
135
// If no ip match this rule, will choose one randomly.
136
DECLARE_String(priority_networks);
137
138
// performance moderate or compact, only tcmalloc compile
139
DECLARE_String(memory_mode);
140
141
// if true, process memory limit and memory usage based on cgroup memory info.
142
DECLARE_mBool(enable_use_cgroup_memory_info);
143
144
// process memory limit specified as number of bytes
145
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
146
// or percentage of the physical memory ('<int>%').
147
// defaults to bytes if no unit is given"
148
// must larger than 0. and if larger than physical memory size,
149
// it will be set to physical memory size.
150
DECLARE_String(mem_limit);
151
152
// Soft memory limit as a fraction of hard memory limit.
153
DECLARE_Double(soft_mem_limit_frac);
154
155
// Schema change memory limit as a fraction of soft memory limit.
156
DECLARE_Double(schema_change_mem_limit_frac);
157
158
// Many modern allocators (for example) do not do a mremap for
159
// realloc, even in case of large enough chunks of memory. Although this allows
160
// you to increase performance and reduce memory consumption during realloc.
161
// To fix this, we do mremap manually if the chunk of memory is large enough.
162
//
163
// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
164
// space is very slow, especially in the case of a large number of threads. We
165
// expect that the set of operations mmap/something to do/mremap can only be
166
// performed about 1000 times per second.
167
//
168
// P.S. This is also required, because tcmalloc can not allocate a chunk of
169
// memory greater than 16 GB.
170
DECLARE_mInt64(mmap_threshold); // bytes
171
172
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
173
// Increase can reduce the number of hash table resize, but may waste more memory.
174
DECLARE_mInt32(hash_table_double_grow_degree);
175
176
// The max fill rate for hash table
177
DECLARE_mInt32(max_fill_rate);
178
179
DECLARE_mInt32(double_resize_threshold);
180
181
// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 6.4G,
182
// actual low water mark=min(6.4G, MemTotal * 5%), avoid wasting too much memory on machines
183
// with large memory larger than 128G.
184
// Turn up max. On machines with more than 128G memory, more memory buffers will be reserved for Full GC.
185
// Turn down max. will use as much memory as possible.
186
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
187
188
// reserve a small amount of memory so we do not trigger MinorGC
189
DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
190
191
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
192
DECLARE_mString(process_minor_gc_size);
193
DECLARE_mString(process_full_gc_size);
194
195
// If true, when the process does not exceed the soft mem limit, the query memory will not be limited;
196
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
197
// used memory and the exec_mem_limit will be canceled.
198
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
199
DECLARE_mBool(enable_query_memory_overcommit);
200
201
// gc will release cache, cancel task, and task will wait for gc to release memory,
202
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
203
DECLARE_mBool(disable_memory_gc);
204
205
// when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G,
206
// if alloc successful, will print a warning with stacktrace, but not prevent memory alloc.
207
// if alloc failed using Doris Allocator, will print stacktrace in error log.
208
// if is -1, disable print stacktrace when alloc large memory.
209
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
210
211
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled.
212
// if you need a core dump to analyze large memory allocation,
213
// modify this parameter to crash when large memory allocation occur will help
214
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
215
216
// If memory tracker value is inaccurate, BE will crash. usually used in test environments, default value is false.
217
DECLARE_mBool(crash_in_memory_tracker_inaccurate);
218
219
// default is true. if any memory tracking in Orphan mem tracker will report error.
220
// !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time.
221
// allocator free memory not need to check, because when the thread memory tracker label is Orphan,
222
// use the tracker saved in Allocator.
223
DECLARE_mBool(enable_memory_orphan_check);
224
225
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
226
DECLARE_mInt32(thread_wait_gc_max_milliseconds);
227
228
// reach mem limit, don't serialize in batch
229
DECLARE_mInt64(pre_serialize_keys_limit_bytes);
230
231
// the port heartbeat service used
232
DECLARE_Int32(heartbeat_service_port);
233
// the count of heart beat service
234
DECLARE_Int32(heartbeat_service_thread_count);
235
// the count of thread to create table
236
DECLARE_Int32(create_tablet_worker_count);
237
// the count of thread to drop table
238
DECLARE_Int32(drop_tablet_worker_count);
239
// the count of thread to batch load
240
DECLARE_Int32(push_worker_count_normal_priority);
241
// the count of thread to high priority batch load
242
DECLARE_Int32(push_worker_count_high_priority);
243
// the count of thread to publish version
244
DECLARE_Int32(publish_version_worker_count);
245
// the count of tablet thread to publish version
246
DECLARE_Int32(tablet_publish_txn_max_thread);
247
// the timeout of EnginPublishVersionTask
248
DECLARE_Int32(publish_version_task_timeout_s);
249
// the count of thread to calc delete bitmap
250
DECLARE_Int32(calc_delete_bitmap_max_thread);
251
// the count of thread to clear transaction task
252
DECLARE_Int32(clear_transaction_task_worker_count);
253
// the count of thread to delete
254
DECLARE_Int32(delete_worker_count);
255
// the count of thread to alter table
256
DECLARE_Int32(alter_tablet_worker_count);
257
// the count of thread to alter index
258
DECLARE_Int32(alter_index_worker_count);
259
// the count of thread to clone
260
DECLARE_Int32(clone_worker_count);
261
// the count of thread to clone
262
DECLARE_Int32(storage_medium_migrate_count);
263
// the count of thread to check consistency
264
DECLARE_Int32(check_consistency_worker_count);
265
// the count of thread to upload
266
DECLARE_Int32(upload_worker_count);
267
// the count of thread to download
268
DECLARE_Int32(download_worker_count);
269
// the count of thread to make snapshot
270
DECLARE_Int32(make_snapshot_worker_count);
271
// the count of thread to release snapshot
272
DECLARE_Int32(release_snapshot_worker_count);
273
// report random wait a little time to avoid FE receiving multiple be reports at the same time.
274
// do not set it to false for production environment
275
DECLARE_mBool(report_random_wait);
276
// the interval time(seconds) for agent report tasks signature to FE
277
DECLARE_mInt32(report_task_interval_seconds);
278
// the interval time(seconds) for refresh storage policy from FE
279
DECLARE_mInt32(storage_refresh_storage_policy_task_interval_seconds);
280
// the interval time(seconds) for agent report disk state to FE
281
DECLARE_mInt32(report_disk_state_interval_seconds);
282
// the interval time(seconds) for agent report olap table to FE
283
DECLARE_mInt32(report_tablet_interval_seconds);
284
// the max download speed(KB/s)
285
DECLARE_mInt32(max_download_speed_kbps);
286
// download low speed limit(KB/s)
287
DECLARE_mInt32(download_low_speed_limit_kbps);
288
// download low speed time(seconds)
289
DECLARE_mInt32(download_low_speed_time);
290
291
// deprecated, use env var LOG_DIR in be.conf
292
DECLARE_String(sys_log_dir);
293
// for udf
294
DECLARE_String(user_function_dir);
295
// INFO, WARNING, ERROR, FATAL
296
DECLARE_String(sys_log_level);
297
// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
298
DECLARE_String(sys_log_roll_mode);
299
// log roll num
300
DECLARE_Int32(sys_log_roll_num);
301
// verbose log
302
DECLARE_Strings(sys_log_verbose_modules);
303
// verbose log level
304
DECLARE_Int32(sys_log_verbose_level);
305
// verbose log FLAGS_v
306
DECLARE_Int32(sys_log_verbose_flags_v);
307
// log buffer level
308
DECLARE_String(log_buffer_level);
309
// log enable custom date time format
310
DECLARE_Bool(sys_log_enable_custom_date_time_format);
311
// log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time)
312
DECLARE_String(sys_log_custom_date_time_format);
313
// log custom date time milliseconds format (fmt::format)
314
DECLARE_String(sys_log_custom_date_time_ms_format);
315
316
// number of threads available to serve backend execution requests
317
DECLARE_Int32(be_service_threads);
318
319
// interval between profile reports; in seconds
320
DECLARE_mInt32(status_report_interval);
321
DECLARE_mInt32(pipeline_status_report_interval);
322
// if true, each disk will have a separate thread pool for scanner
323
DECLARE_Bool(doris_enable_scanner_thread_pool_per_disk);
324
// the timeout of a work thread to wait the blocking priority queue to get a task
325
DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
326
// number of scanner thread pool size for olap table
327
// and the min thread num of remote scanner thread pool
328
DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
329
DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num);
330
// number of batch size to fetch the remote split source
331
DECLARE_mInt32(remote_split_source_batch_size);
332
// max number of remote scanner thread pool size
333
// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
334
DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
335
// number of olap scanner thread pool queue size
336
DECLARE_Int32(doris_scanner_thread_pool_queue_size);
337
// default thrift client connect timeout(in seconds)
338
DECLARE_mInt32(thrift_connect_timeout_seconds);
339
DECLARE_mInt32(fetch_rpc_timeout_seconds);
340
// default thrift client retry interval (in milliseconds)
341
DECLARE_mInt64(thrift_client_retry_interval_ms);
342
// max message size of thrift request
343
// default: 100 * 1024 * 1024
344
DECLARE_mInt64(thrift_max_message_size);
345
// max row count number for single scan range, used in segmentv1
346
DECLARE_mInt32(doris_scan_range_row_count);
347
// max bytes number for single scan range, used in segmentv2
348
DECLARE_mInt32(doris_scan_range_max_mb);
349
// max bytes number for single scan block, used in segmentv2
350
DECLARE_mInt32(doris_scan_block_max_mb);
351
// size of scanner queue between scanner thread and compute thread
352
DECLARE_mInt32(doris_scanner_queue_size);
353
// single read execute fragment row number
354
DECLARE_mInt32(doris_scanner_row_num);
355
// single read execute fragment row bytes
356
DECLARE_mInt32(doris_scanner_row_bytes);
357
// single read execute fragment max run time millseconds
358
DECLARE_mInt32(doris_scanner_max_run_time_ms);
359
DECLARE_mInt32(min_bytes_in_scanner_queue);
360
// number of max scan keys
361
DECLARE_mInt32(doris_max_scan_key_num);
362
// the max number of push down values of a single column.
363
// if exceed, no conditions will be pushed down for that column.
364
DECLARE_mInt32(max_pushdown_conditions_per_column);
365
// (Advanced) Maximum size of per-query receive-side buffer
366
DECLARE_mInt32(exchg_node_buffer_size_bytes);
367
DECLARE_mInt32(exchg_buffer_queue_capacity_factor);
368
369
DECLARE_mInt64(column_dictionary_key_ratio_threshold);
370
DECLARE_mInt64(column_dictionary_key_size_threshold);
371
// memory_limitation_per_thread_for_schema_change_bytes unit bytes
372
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
373
DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);
374
375
// all cache prune interval, used by GC and periodic thread.
376
DECLARE_mInt32(cache_prune_interval_sec);
377
DECLARE_mInt32(cache_periodic_prune_stale_sweep_sec);
378
// the clean interval of tablet lookup cache
379
DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec);
380
DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec);
381
DECLARE_mInt32(disk_stat_monitor_interval);
382
DECLARE_mInt32(unused_rowset_monitor_interval);
383
DECLARE_String(storage_root_path);
384
DECLARE_mString(broken_storage_path);
385
386
// Config is used to check incompatible old format hdr_ format
387
// whether doris uses strict way. When config is true, process will log fatal
388
// and exit. When config is false, process will only log warning.
389
DECLARE_Bool(storage_strict_check_incompatible_old_format);
390
391
// BE process will exit if the percentage of error disk reach this value.
392
DECLARE_mInt32(max_percentage_of_error_disk);
393
DECLARE_mInt32(default_num_rows_per_column_file_block);
394
// pending data policy
395
DECLARE_mInt32(pending_data_expire_time_sec);
396
// inc_rowset snapshot rs sweep time interval
397
DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec);
398
// tablet stale rowset sweep by threshold size
399
DECLARE_Bool(tablet_rowset_stale_sweep_by_size);
400
DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size);
401
// garbage sweep policy
402
DECLARE_Int32(max_garbage_sweep_interval);
403
DECLARE_Int32(min_garbage_sweep_interval);
404
// garbage sweep every batch will sleep 1ms
405
DECLARE_mInt32(garbage_sweep_batch_size);
406
DECLARE_mInt32(snapshot_expire_time_sec);
407
// It is only a recommended value. When the disk space is insufficient,
408
// the file storage period under trash dose not have to comply with this parameter.
409
DECLARE_mInt32(trash_file_expire_time_sec);
410
// minimum file descriptor number
411
// modify them upon necessity
412
DECLARE_Int32(min_file_descriptor_number);
413
DECLARE_mBool(disable_segment_cache);
414
DECLARE_String(row_cache_mem_limit);
415
416
// Cache for storage page size
417
DECLARE_String(storage_page_cache_limit);
418
// Shard size for page cache, the value must be power of two.
419
// It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions.
420
DECLARE_Int32(storage_page_cache_shard_size);
421
// Percentage for index page cache
422
// all storage page cache will be divided into data_page_cache and index_page_cache
423
DECLARE_Int32(index_page_cache_percentage);
424
// whether to disable page cache feature in storage
425
// TODO delete it. Divided into Data page, Index page, pk index page
426
DECLARE_Bool(disable_storage_page_cache);
427
// whether to disable row cache feature in storage
428
DECLARE_mBool(disable_storage_row_cache);
429
// whether to disable pk page cache feature in storage
430
DECLARE_Bool(disable_pk_storage_page_cache);
431
432
// Cache for mow primary key storage page size, it's seperated from
433
// storage_page_cache_limit
434
DECLARE_String(pk_storage_page_cache_limit);
435
// data page size for primary key index
436
DECLARE_Int32(primary_key_data_page_size);
437
438
// inc_rowset snapshot rs sweep time interval
439
DECLARE_mInt32(data_page_cache_stale_sweep_time_sec);
440
DECLARE_mInt32(index_page_cache_stale_sweep_time_sec);
441
// great impact on the performance of MOW, so it can be longer.
442
DECLARE_mInt32(pk_index_page_cache_stale_sweep_time_sec);
443
444
DECLARE_Bool(enable_low_cardinality_optimize);
445
DECLARE_Bool(enable_low_cardinality_cache_code);
446
447
// be policy
448
// whether check compaction checksum
449
DECLARE_mBool(enable_compaction_checksum);
450
// whether disable automatic compaction task
451
DECLARE_mBool(disable_auto_compaction);
452
// whether enable vertical compaction
453
DECLARE_mBool(enable_vertical_compaction);
454
// whether enable ordered data compaction
455
DECLARE_mBool(enable_ordered_data_compaction);
456
// In vertical compaction, column number for every group
457
DECLARE_mInt32(vertical_compaction_num_columns_per_group);
458
// In vertical compaction, max memory usage for row_source_buffer
459
DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
460
// In vertical compaction, max dest segment file size
461
DECLARE_mInt64(vertical_compaction_max_segment_size);
462
463
// If enabled, segments will be flushed column by column
464
DECLARE_mBool(enable_vertical_segment_writer);
465
466
// In ordered data compaction, min segment size for input rowset
467
DECLARE_mInt32(ordered_data_compaction_min_segment_size);
468
469
// This config can be set to limit thread number in compaction thread pool.
470
DECLARE_mInt32(max_base_compaction_threads);
471
DECLARE_mInt32(max_cumu_compaction_threads);
472
DECLARE_mInt32(max_single_replica_compaction_threads);
473
474
DECLARE_Bool(enable_base_compaction_idle_sched);
475
DECLARE_mInt64(base_compaction_min_rowset_num);
476
DECLARE_mInt64(base_compaction_max_compaction_score);
477
DECLARE_mDouble(base_compaction_min_data_ratio);
478
DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes);
479
480
DECLARE_Bool(enable_skip_tablet_compaction);
481
// output rowset of cumulative compaction total disk size exceed this config size,
482
// this rowset will be given to base compaction, unit is m byte.
483
DECLARE_mInt64(compaction_promotion_size_mbytes);
484
485
// output rowset of cumulative compaction total disk size exceed this config ratio of
486
// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
487
// 0 and 1.
488
DECLARE_mDouble(compaction_promotion_ratio);
489
490
// the smallest size of rowset promotion. When the rowset is less than this config, this
491
// rowset will be not given to base compaction. The unit is m byte.
492
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
493
494
// When output rowset of cumulative compaction total version count (end_version - start_version)
495
// exceed this config count, the rowset will be moved to base compaction
496
// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
497
// related cost on delete bitmap more effectively.
498
DECLARE_mInt64(compaction_promotion_version_count);
499
500
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
501
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
502
DECLARE_mInt64(compaction_min_size_mbytes);
503
504
// cumulative compaction policy: min and max delta file's number
505
DECLARE_mInt64(cumulative_compaction_min_deltas);
506
DECLARE_mInt64(cumulative_compaction_max_deltas);
507
DECLARE_mInt32(cumulative_compaction_max_deltas_factor);
508
509
// This config can be set to limit thread number in  multiget thread pool.
510
DECLARE_mInt32(multi_get_max_threads);
511
512
// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
513
DECLARE_mInt64(total_permits_for_compaction_score);
514
515
// sleep interval in ms after generated compaction tasks
516
DECLARE_mInt32(generate_compaction_tasks_interval_ms);
517
// sleep interval in second after update replica infos
518
DECLARE_mInt32(update_replica_infos_interval_seconds);
519
520
// Compaction task number per disk.
521
// Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each.
522
DECLARE_mInt32(compaction_task_num_per_disk);
523
// compaction thread num for fast disk(typically .SSD), must be greater than 2.
524
DECLARE_mInt32(compaction_task_num_per_fast_disk);
525
526
// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
527
DECLARE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round);
528
529
// Not compact the invisible versions, but with some limitations:
530
// if not timeout, keep no more than compaction_keep_invisible_version_max_count versions;
531
// if timeout, keep no more than compaction_keep_invisible_version_min_count versions.
532
DECLARE_mInt32(compaction_keep_invisible_version_timeout_sec);
533
DECLARE_mInt32(compaction_keep_invisible_version_min_count);
534
DECLARE_mInt32(compaction_keep_invisible_version_max_count);
535
536
// Threshold to logging compaction trace, in seconds.
537
DECLARE_mInt32(base_compaction_trace_threshold);
538
DECLARE_mInt32(cumulative_compaction_trace_threshold);
539
DECLARE_mBool(disable_compaction_trace_log);
540
541
// Interval to picking rowset to compact, in seconds
542
DECLARE_mInt64(pick_rowset_to_compact_interval_sec);
543
544
// Compaction priority schedule
545
DECLARE_mBool(enable_compaction_priority_scheduling);
546
DECLARE_mInt32(low_priority_compaction_task_num_per_disk);
547
DECLARE_mInt32(low_priority_compaction_score_threshold);
548
549
// Thread count to do tablet meta checkpoint, -1 means use the data directories count.
550
DECLARE_Int32(max_meta_checkpoint_threads);
551
552
// Threshold to logging agent task trace, in seconds.
553
DECLARE_mInt32(agent_task_trace_threshold_sec);
554
555
// This config can be set to limit thread number in tablet migration thread pool.
556
DECLARE_Int32(min_tablet_migration_threads);
557
DECLARE_Int32(max_tablet_migration_threads);
558
559
DECLARE_mInt32(finished_migration_tasks_size);
560
// If size less than this, the remaining rowsets will be force to complete
561
DECLARE_mInt32(migration_remaining_size_threshold_mb);
562
// If the task runs longer than this time, the task will be terminated, in seconds.
563
// timeout = std::max(migration_task_timeout_secs,  tablet size / 1MB/s)
564
DECLARE_mInt32(migration_task_timeout_secs);
565
// timeout for try_lock migration lock
566
DECLARE_Int64(migration_lock_timeout_ms);
567
568
// Port to start debug webserver on
569
DECLARE_Int32(webserver_port);
570
// Https enable flag
571
DECLARE_Bool(enable_https);
572
// Path of certificate
573
DECLARE_String(ssl_certificate_path);
574
// Path of private key
575
DECLARE_String(ssl_private_key_path);
576
// Whether to check authorization
577
DECLARE_Bool(enable_all_http_auth);
578
// Number of webserver workers
579
DECLARE_Int32(webserver_num_workers);
580
581
DECLARE_Bool(enable_single_replica_load);
582
// Number of download workers for single replica load
583
DECLARE_Int32(single_replica_load_download_num_workers);
584
585
// Used for mini Load. mini load data file will be removed after this time.
586
DECLARE_Int64(load_data_reserve_hours);
587
// log error log will be removed after this time
588
DECLARE_mInt64(load_error_log_reserve_hours);
589
// error log size limit, default 200MB
590
DECLARE_mInt64(load_error_log_limit_bytes);
591
592
// be brpc interface is classified into two categories: light and heavy
593
// each category has diffrent thread number
594
// threads to handle heavy api interface, such as transmit_data/transmit_block etc
595
// Default, if less than or equal 32 core, the following are 128, 128, 10240, 10240 in turn.
596
//          if greater than 32 core, the following are core num * 4, core num * 4, core num * 320, core num * 320 in turn
597
DECLARE_Int32(brpc_heavy_work_pool_threads);
598
// threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start
599
DECLARE_Int32(brpc_light_work_pool_threads);
600
DECLARE_Int32(brpc_heavy_work_pool_max_queue_size);
601
DECLARE_Int32(brpc_light_work_pool_max_queue_size);
602
DECLARE_Int32(brpc_arrow_flight_work_pool_threads);
603
DECLARE_Int32(brpc_arrow_flight_work_pool_max_queue_size);
604
605
// The maximum amount of data that can be processed by a stream load
606
DECLARE_mInt64(streaming_load_max_mb);
607
// Some data formats, such as JSON, cannot be streamed.
608
// Therefore, it is necessary to limit the maximum number of
609
// such data when using stream load to prevent excessive memory consumption.
610
DECLARE_mInt64(streaming_load_json_max_mb);
611
// the alive time of a TabletsChannel.
612
// If the channel does not receive any data till this time,
613
// the channel will be removed.
614
DECLARE_mInt32(streaming_load_rpc_max_alive_time_sec);
615
// the timeout of a rpc to open the tablet writer in remote BE.
616
// short operation time, can set a short timeout
617
DECLARE_Int32(tablet_writer_open_rpc_timeout_sec);
618
// You can ignore brpc error '[E1011]The server is overcrowded' when writing data.
619
DECLARE_mBool(tablet_writer_ignore_eovercrowded);
620
DECLARE_mBool(exchange_sink_ignore_eovercrowded);
621
DECLARE_mInt32(slave_replica_writer_rpc_timeout_sec);
622
// Whether to enable stream load record function, the default is false.
623
// False: disable stream load record
624
DECLARE_mBool(enable_stream_load_record);
625
// batch size of stream load record reported to FE
626
DECLARE_mInt32(stream_load_record_batch_size);
627
// expire time of stream load record in rocksdb.
628
DECLARE_Int32(stream_load_record_expire_time_secs);
629
// time interval to clean expired stream load records
630
DECLARE_mInt64(clean_stream_load_record_interval_secs);
631
// The buffer size to store stream table function schema info
632
DECLARE_Int64(stream_tvf_buffer_size);
633
634
// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
635
// You may need to lower the speed when the sink receiver bes are too busy.
636
DECLARE_mInt32(olap_table_sink_send_interval_microseconds);
637
// For auto partition, the send interval will multiply the factor
638
DECLARE_mDouble(olap_table_sink_send_interval_auto_partition_factor);
639
640
// Fragment thread pool
641
DECLARE_Int32(fragment_pool_thread_num_min);
642
DECLARE_Int32(fragment_pool_thread_num_max);
643
DECLARE_Int32(fragment_pool_queue_size);
644
645
// Control the number of disks on the machine.  If 0, this comes from the system settings.
646
DECLARE_Int32(num_disks);
647
// The maximum number of the threads per disk is also the max queue depth per disk.
648
DECLARE_Int32(num_threads_per_disk);
649
// The read size is the size of the reads sent to os.
650
// There is a trade off of latency and throughout, trying to keep disks busy but
651
// not introduce seeks.  The literature seems to agree that with 8 MB reads, random
652
// io and sequential io perform similarly.
653
DECLARE_Int32(read_size);       // 8 * 1024 * 1024, Read Size (in bytes)
654
DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
655
656
// for pprof
657
DECLARE_String(pprof_profile_dir);
658
// for jeprofile in jemalloc
659
DECLARE_mString(jeprofile_dir);
660
// Purge all unused dirty pages for all arenas.
661
DECLARE_mBool(enable_je_purge_dirty_pages);
662
// Purge all unused Jemalloc dirty pages for all arenas when exceed je_dirty_pages_mem_limit and process exceed soft limit.
663
DECLARE_mString(je_dirty_pages_mem_limit_percent);
664
665
// to forward compatibility, will be removed later
666
DECLARE_mBool(enable_token_check);
667
668
// to open/close system metrics
669
DECLARE_Bool(enable_system_metrics);
670
671
// Number of cores Doris will used, this will effect only when it's greater than 0.
672
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
673
DECLARE_Int32(num_cores);
674
675
// When BE start, If there is a broken disk, BE process will exit by default.
676
// Otherwise, we will ignore the broken disk,
677
DECLARE_Bool(ignore_broken_disk);
678
679
// Sleep time in milliseconds between memory maintenance iterations
680
DECLARE_mInt32(memory_maintenance_sleep_time_ms);
681
682
// After full gc, no longer full gc and minor gc during sleep.
683
// After minor gc, no minor gc during sleep, but full gc is possible.
684
DECLARE_mInt32(memory_gc_sleep_time_ms);
685
686
// Sleep time in milliseconds between memtbale flush mgr memory refresh iterations
687
DECLARE_mInt64(memtable_mem_tracker_refresh_interval_ms);
688
689
// Sleep time in milliseconds between refresh iterations of workload group weighted memory ratio
690
DECLARE_mInt64(wg_weighted_memory_ratio_refresh_interval_ms);
691
692
// percent of (active memtables size / all memtables size) when reach hard limit
693
DECLARE_mInt32(memtable_hard_limit_active_percent);
694
695
// percent of (active memtables size / all memtables size) when reach soft limit
696
DECLARE_mInt32(memtable_soft_limit_active_percent);
697
698
// memtable insert memory tracker will multiply input block size with this ratio
699
DECLARE_mDouble(memtable_insert_memory_ratio);
700
// max write buffer size before flush, default 200MB
701
DECLARE_mInt64(write_buffer_size);
702
// max buffer size used in memtable for the aggregated table, default 400MB
703
DECLARE_mInt64(write_buffer_size_for_agg);
704
// max parallel flush task per memtable writer
705
DECLARE_mInt32(memtable_flush_running_count_limit);
706
707
DECLARE_Int32(load_process_max_memory_limit_percent); // 50%
708
709
// If the memory consumption of load jobs exceed load_process_max_memory_limit,
710
// all load jobs will hang there to wait for memtable flush. We should have a
711
// soft limit which can trigger the memtable flush for the load channel who
712
// consumes lagest memory size before we reach the hard limit. The soft limit
713
// might avoid all load jobs hang at the same time.
714
DECLARE_Int32(load_process_soft_mem_limit_percent);
715
716
// If load memory consumption is within load_process_safe_mem_permit_percent,
717
// memtable memory limiter will do nothing.
718
DECLARE_Int32(load_process_safe_mem_permit_percent);
719
720
// result buffer cancelled time (unit: second)
721
DECLARE_mInt32(result_buffer_cancelled_interval_time);
722
723
// arrow flight result sink buffer rows size, default 4096 * 8
724
DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows);
725
// The timeout for ADBC Client to wait for data using arrow flight reader.
726
// If the query is very complex and no result is generated after this time, consider increasing this timeout.
727
DECLARE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms);
728
729
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
730
DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency);
731
732
// sync tablet_meta when modifying meta
733
DECLARE_mBool(sync_tablet_meta);
734
735
// default thrift rpc timeout ms
736
DECLARE_mInt32(thrift_rpc_timeout_ms);
737
738
// txn commit rpc timeout
739
DECLARE_mInt32(txn_commit_rpc_timeout_ms);
740
741
// If set to true, metric calculator will run
742
DECLARE_Bool(enable_metric_calculator);
743
744
// max consumer num in one data consumer group, for routine load
745
DECLARE_mInt32(max_consumer_num_per_group);
746
747
// the max size of thread pool for routine load task.
748
// this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5)
749
DECLARE_Int32(max_routine_load_thread_pool_size);
750
751
// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
752
// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
753
DECLARE_mInt32(max_memory_sink_batch_count);
754
755
// This configuration is used for the context gc thread schedule period
756
// note: unit is minute, default is 5min
757
DECLARE_mInt32(scan_context_gc_interval_min);
758
759
// es scroll keep-alive
760
DECLARE_String(es_scroll_keepalive);
761
762
// HTTP connection timeout for es
763
DECLARE_mInt32(es_http_timeout_ms);
764
765
// the max client cache number per each host
766
// There are variety of client cache in BE, but currently we use the
767
// same cache size configuration.
768
// TODO(cmy): use different config to set different client cache if necessary.
769
DECLARE_Int32(max_client_cache_size_per_host);
770
771
// Dir to save files downloaded by SmallFileMgr
772
DECLARE_String(small_file_dir);
773
// path gc
774
DECLARE_Bool(path_gc_check);
775
DECLARE_mInt32(path_gc_check_interval_second);
776
DECLARE_mInt32(path_gc_check_step);
777
DECLARE_mInt32(path_gc_check_step_interval_ms);
778
779
// The following 2 configs limit the max usage of disk capacity of a data dir.
780
// If both of these 2 threshold reached, no more data can be writen into that data dir.
781
// The percent of max used capacity of a data dir
782
DECLARE_mInt32(storage_flood_stage_usage_percent); // 90%
783
// The min bytes that should be left of a data dir
784
DECLARE_mInt64(storage_flood_stage_left_capacity_bytes); // 1GB
785
// number of thread for flushing memtable per store
786
DECLARE_Int32(flush_thread_num_per_store);
787
// number of thread for flushing memtable per store, for high priority load task
788
DECLARE_Int32(high_priority_flush_thread_num_per_store);
789
790
// workload group's flush thread num
791
DECLARE_Int32(wg_flush_thread_num_per_store);
792
793
// config for tablet meta checkpoint
794
DECLARE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num);
795
DECLARE_mInt32(tablet_meta_checkpoint_min_interval_secs);
796
DECLARE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs);
797
798
// config for default rowset type
799
// Valid configs: ALPHA, BETA
800
DECLARE_String(default_rowset_type);
801
802
// Maximum size of a single message body in all protocols
803
DECLARE_Int64(brpc_max_body_size);
804
// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
805
// Default, if the physical memory is less than or equal to 64G, the value is 1G
806
//          if the physical memory is greater than 64G, the value is physical memory * mem_limit(0.8) / 1024 * 20
807
DECLARE_Int64(brpc_socket_max_unwritten_bytes);
808
// TODO(zxy): expect to be true in v1.3
809
// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
810
// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
811
// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
812
DECLARE_mBool(transfer_large_data_by_brpc);
813
814
// max number of txns for every txn_partition_map in txn manager
815
// this is a self protection to avoid too many txns saving in manager
816
DECLARE_mInt64(max_runnings_transactions_per_txn_map);
817
818
// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
819
// this is a an enhancement for better performance to manage tablet
820
DECLARE_Int32(tablet_map_shard_size);
821
822
// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
823
// this is a an enhancement for better performance to manage txn
824
DECLARE_Int32(txn_map_shard_size);
825
826
// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
827
// this is a an enhancement for better performance to commit and publish txn
828
DECLARE_Int32(txn_shard_size);
829
830
// Whether to continue to start be when load tablet from header failed.
831
DECLARE_Bool(ignore_load_tablet_failure);
832
833
// Whether to continue to start be when load tablet from header failed.
834
DECLARE_mBool(ignore_rowset_stale_unconsistent_delete);
835
836
// Set max cache's size of query results, the unit is M byte
837
DECLARE_Int32(query_cache_max_size_mb);
838
839
// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
840
DECLARE_Int32(query_cache_elasticity_size_mb);
841
842
// Maximum number of cache partitions corresponding to a SQL
843
DECLARE_Int32(query_cache_max_partition_count);
844
845
// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
846
// the load process will reject new incoming load job of this tablet.
847
// This is to avoid too many version num.
848
DECLARE_mInt32(max_tablet_version_num);
849
850
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
851
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
852
DECLARE_String(thrift_server_type_of_fe);
853
854
// disable zone map index when page row is too few
855
DECLARE_mInt32(zone_map_row_num_threshold);
856
857
// aws sdk log level
858
//    Off = 0,
859
//    Fatal = 1,
860
//    Error = 2,
861
//    Warn = 3,
862
//    Info = 4,
863
//    Debug = 5,
864
//    Trace = 6
865
DECLARE_Int32(aws_log_level);
866
867
// the buffer size when read data from remote storage like s3
868
DECLARE_mInt32(remote_storage_read_buffer_mb);
869
870
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
871
// smaller than this value will continue to accumulate. specified as number of bytes.
872
// Decreasing this value will increase the frequency of consume/release.
873
// Increasing this value will cause MemTracker statistics to be inaccurate.
874
DECLARE_mInt32(mem_tracker_consume_min_size_bytes);
875
876
// The version information of the tablet will be stored in the memory
877
// in an adjacency graph data structure.
878
// And as the new version is written and the old version is deleted,
879
// the data structure will begin to have empty vertex with no edge associations(orphan vertex).
880
// This config is used to control that when the proportion of orphan vertex is greater than the threshold,
881
// the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely.
882
// This config usually only needs to be modified during testing.
883
// In most cases, it does not need to be modified.
884
DECLARE_mDouble(tablet_version_graph_orphan_vertex_ratio);
885
886
// share delta writers when memtable_on_sink_node = true
887
DECLARE_Bool(share_delta_writers);
888
// timeout for open load stream rpc in ms
889
DECLARE_Int64(open_load_stream_timeout_ms);
890
// enable write background when using brpc stream
891
DECLARE_mBool(enable_brpc_stream_write_background);
892
893
// brpc streaming max_buf_size in bytes
894
DECLARE_Int64(load_stream_max_buf_size);
895
// brpc streaming messages_in_batch
896
DECLARE_Int32(load_stream_messages_in_batch);
897
// brpc streaming StreamWait seconds on EAGAIN
898
DECLARE_Int32(load_stream_eagain_wait_seconds);
899
// max tasks per flush token in load stream
900
DECLARE_Int32(load_stream_flush_token_max_tasks);
901
// max wait flush token time in load stream
902
DECLARE_Int32(load_stream_max_wait_flush_token_time_ms);
903
904
// max send batch parallelism for OlapTableSink
905
// The value set by the user for send_batch_parallelism is not allowed to exceed max_send_batch_parallelism_per_job,
906
// if exceed, the value of send_batch_parallelism would be max_send_batch_parallelism_per_job
907
DECLARE_mInt32(max_send_batch_parallelism_per_job);
908
909
// number of send batch thread pool size
910
DECLARE_Int32(send_batch_thread_pool_thread_num);
911
// number of send batch thread pool queue size
912
DECLARE_Int32(send_batch_thread_pool_queue_size);
913
914
// Limit the number of segment of a newly created rowset.
915
// The newly created rowset may to be compacted after loading,
916
// so if there are too many segment in a rowset, the compaction process
917
// will run out of memory.
918
// When doing compaction, each segment may take at least 1MB buffer.
919
DECLARE_mInt32(max_segment_num_per_rowset);
920
921
// Store segment without compression if a segment is smaller than
922
// segment_compression_threshold_kb.
923
DECLARE_mInt32(segment_compression_threshold_kb);
924
925
// The connection timeout when connecting to external table such as odbc table.
926
DECLARE_mInt32(external_table_connect_timeout_sec);
927
928
// Time to clean up useless JDBC connection pool cache
929
DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
930
931
// Global bitmap cache capacity for aggregation cache, size in bytes
932
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
933
DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
934
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
935
936
// A common object cache depends on an Sharded LRU Cache.
937
DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
938
939
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
940
// If the dependent kafka broker version older than 0.10.0.0,
941
// the value of kafka_api_version_request should be false, and the
942
// value set by the fallback version kafka_broker_version_fallback will be used,
943
// and the valid values are: 0.9.0.x, 0.8.x.y.
944
DECLARE_String(kafka_api_version_request);
945
DECLARE_String(kafka_broker_version_fallback);
946
DECLARE_mString(kafka_debug);
947
948
// The number of pool siz of routine load consumer.
949
// If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608
950
// Change this size to 0 to fix it temporarily.
951
DECLARE_mInt32(routine_load_consumer_pool_size);
952
953
// Used in single-stream-multi-table load. When receive a batch of messages from kafka,
954
// if the size of batch is more than this threshold, we will request plans for all related tables.
955
DECLARE_Int32(multi_table_batch_plan_threshold);
956
957
// Used in single-stream-multi-table load. When receiving a batch of messages from Kafka,
958
// if the size of the table wait for plan is more than this threshold, we will request plans for all related tables.
959
// The param is aimed to avoid requesting and executing too many plans at once.
960
// Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC
961
// and improve the real-time processing of data.
962
DECLARE_Int32(multi_table_max_wait_tables);
963
964
// When the timeout of a load task is less than this threshold,
965
// Doris treats it as a high priority task.
966
// high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic.
967
// this threshold is mainly used to identify routine load tasks and should not be modified if not necessary.
968
DECLARE_mInt32(load_task_high_priority_threshold_second);
969
970
// The min timeout of load rpc (add batch, close)
971
// Because a load rpc may be blocked for a while.
972
// Increase this config may avoid rpc timeout.
973
DECLARE_mInt32(min_load_rpc_timeout_ms);
974
975
// use which protocol to access function service, candicate is baidu_std/h2:grpc
976
DECLARE_String(function_service_protocol);
977
978
// use which load balancer to select server to connect
979
DECLARE_String(rpc_load_balancer);
980
981
// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
982
// so we set a soft limit, default is 1MB
983
DECLARE_mInt32(string_type_length_soft_limit_bytes);
984
985
DECLARE_mInt32(jsonb_type_length_soft_limit_bytes);
986
987
// Threshold fo reading a small file into memory
988
DECLARE_mInt32(in_memory_file_size);
989
990
// ParquetReaderWrap prefetch buffer size
991
DECLARE_Int32(parquet_reader_max_buffer_size);
992
// Max size of parquet page header in bytes
993
DECLARE_mInt32(parquet_header_max_size_mb);
994
// Max buffer size for parquet row group
995
DECLARE_mInt32(parquet_rowgroup_max_buffer_mb);
996
// Max buffer size for parquet chunk column
997
DECLARE_mInt32(parquet_column_max_buffer_mb);
998
// Merge small IO, the max amplified read ratio
999
DECLARE_mDouble(max_amplified_read_ratio);
1000
// Equivalent min size of each IO that can reach the maximum storage speed limit
1001
// 1MB for oss, 8KB for hdfs
1002
DECLARE_mInt32(merged_oss_min_io_size);
1003
DECLARE_mInt32(merged_hdfs_min_io_size);
1004
1005
// OrcReader
1006
DECLARE_mInt32(orc_natural_read_size_mb);
1007
DECLARE_mInt64(big_column_size_buffer);
1008
DECLARE_mInt64(small_column_size_buffer);
1009
1010
// When the rows number reached this limit, will check the filter rate the of bloomfilter
1011
// if it is lower than a specific threshold, the predicate will be disabled.
1012
DECLARE_mInt32(rf_predicate_check_row_num);
1013
1014
// cooldown task configs
1015
DECLARE_Int32(cooldown_thread_num);
1016
DECLARE_mInt64(generate_cooldown_task_interval_sec);
1017
DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h
1018
DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
1019
DECLARE_Int32(cold_data_compaction_thread_num);
1020
DECLARE_mInt32(cold_data_compaction_interval_sec);
1021
1022
DECLARE_Int32(s3_transfer_executor_pool_size);
1023
1024
DECLARE_Bool(enable_time_lut);
1025
DECLARE_mBool(enable_simdjson_reader);
1026
1027
DECLARE_mBool(enable_query_like_bloom_filter);
1028
// number of s3 scanner thread pool size
1029
DECLARE_Int32(doris_remote_scanner_thread_pool_thread_num);
1030
// number of s3 scanner thread pool queue size
1031
DECLARE_Int32(doris_remote_scanner_thread_pool_queue_size);
1032
1033
// limit the queue of pending batches which will be sent by a single nodechannel
1034
DECLARE_mInt64(nodechannel_pending_queue_max_bytes);
1035
1036
// The batch size for sending data by brpc streaming client
1037
DECLARE_mInt64(brpc_streaming_client_batch_bytes);
1038
1039
DECLARE_Bool(enable_brpc_builtin_services);
1040
1041
DECLARE_Bool(enable_brpc_connection_check);
1042
1043
// Max waiting time to wait the "plan fragment start" rpc.
1044
// If timeout, the fragment will be cancelled.
1045
// This parameter is usually only used when the FE loses connection,
1046
// and the BE can automatically cancel the relevant fragment after the timeout,
1047
// so as to avoid occupying the execution thread for a long time.
1048
DECLARE_mInt32(max_fragment_start_wait_time_seconds);
1049
1050
DECLARE_Int32(fragment_mgr_cancel_worker_interval_seconds);
1051
1052
// Node role tag for backend. Mix role is the default role, and computation role have no
1053
// any tablet.
1054
DECLARE_String(be_node_role);
1055
1056
// Hide webserver page for safety.
1057
// Hide the be config page for webserver.
1058
DECLARE_Bool(hide_webserver_config_page);
1059
1060
DECLARE_Bool(enable_segcompaction);
1061
1062
// Max number of segments allowed in a single segcompaction task.
1063
DECLARE_Int32(segcompaction_batch_size);
1064
1065
// Max row count allowed in a single source segment, bigger segments will be skipped.
1066
DECLARE_Int32(segcompaction_candidate_max_rows);
1067
1068
// Max file size allowed in a single source segment, bigger segments will be skipped.
1069
DECLARE_Int64(segcompaction_candidate_max_bytes);
1070
1071
// Max total row count allowed in a single segcompaction task.
1072
DECLARE_Int32(segcompaction_task_max_rows);
1073
1074
// Max total file size allowed in a single segcompaction task.
1075
DECLARE_Int64(segcompaction_task_max_bytes);
1076
1077
// Global segcompaction thread pool size.
1078
DECLARE_mInt32(segcompaction_num_threads);
1079
1080
// enable java udf and jdbc scannode
1081
DECLARE_Bool(enable_java_support);
1082
1083
// Set config randomly to check more issues in github workflow
1084
DECLARE_Bool(enable_fuzzy_mode);
1085
1086
DECLARE_Bool(enable_debug_points);
1087
1088
DECLARE_Int32(pipeline_executor_size);
1089
1090
// block file cache
1091
DECLARE_Bool(enable_file_cache);
1092
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
1093
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
1094
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,"normal_percent":85, "disposable_percent":10, "index_percent":5}]
1095
DECLARE_String(file_cache_path);
1096
DECLARE_Int64(async_file_cache_init_file_num_interval);
1097
DECLARE_Int64(async_file_cache_init_sleep_interval_ms);
1098
DECLARE_Int64(file_cache_min_file_segment_size);
1099
DECLARE_Int64(file_cache_max_file_segment_size);
1100
DECLARE_Bool(clear_file_cache);
1101
DECLARE_Bool(enable_file_cache_query_limit);
1102
// only for debug, will be removed after finding out the root cause
1103
DECLARE_mInt32(file_cache_wait_sec_after_fail); // zero for no waiting and retrying
1104
DECLARE_mInt32(file_cache_max_evict_num_per_round);
1105
1106
// inverted index searcher cache
1107
// cache entry stay time after lookup
1108
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
1109
// cache entry that have not been visited for a certain period of time can be cleaned up by GC thread
1110
DECLARE_mInt32(inverted_index_cache_stale_sweep_time_sec);
1111
// inverted index searcher cache size
1112
DECLARE_String(inverted_index_searcher_cache_limit);
1113
// set `true` to enable insert searcher into cache when write inverted index data
1114
DECLARE_Bool(enable_write_index_searcher_cache);
1115
DECLARE_Bool(enable_inverted_index_cache_check_timestamp);
1116
DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50%
1117
DECLARE_Int32(inverted_index_query_cache_shards);
1118
1119
// inverted index match bitmap cache size
1120
DECLARE_String(inverted_index_query_cache_limit);
1121
1122
// inverted index
1123
DECLARE_mDouble(inverted_index_ram_buffer_size);
1124
DECLARE_mInt32(inverted_index_max_buffered_docs);
1125
// dict path for chinese analyzer
1126
DECLARE_String(inverted_index_dict_path);
1127
DECLARE_Int32(inverted_index_read_buffer_size);
1128
// tree depth for bkd index
1129
DECLARE_Int32(max_depth_in_bkd_tree);
1130
// index compaction
1131
DECLARE_mBool(inverted_index_compaction_enable);
1132
// Only for debug, do not use in production
1133
DECLARE_mBool(debug_inverted_index_compaction);
1134
// index by RAM directory
1135
DECLARE_mBool(inverted_index_ram_dir_enable);
1136
// use num_broadcast_buffer blocks as buffer to do broadcast
1137
DECLARE_Int32(num_broadcast_buffer);
1138
1139
// max depth of expression tree allowed.
1140
DECLARE_Int32(max_depth_of_expr_tree);
1141
1142
// Report a tablet as bad when io errors occurs more than this value.
1143
DECLARE_mInt64(max_tablet_io_errors);
1144
1145
// Report a tablet as bad when its path not found
1146
DECLARE_Int32(tablet_path_check_interval_seconds);
1147
DECLARE_mInt32(tablet_path_check_batch_size);
1148
1149
// it must be larger than or equal to 5MB
1150
DECLARE_mInt32(s3_write_buffer_size);
1151
// The timeout config for S3 buffer allocation
1152
DECLARE_mInt32(s3_writer_buffer_allocation_timeout);
1153
// the max number of cached file handle for block segemnt
1154
DECLARE_mInt64(file_cache_max_file_reader_cache_size);
1155
//enable shrink memory
1156
DECLARE_mBool(enable_shrink_memory);
1157
// enable cache for high concurrent point query work load
1158
DECLARE_mInt32(schema_cache_capacity);
1159
DECLARE_mInt32(schema_cache_sweep_time_sec);
1160
1161
// max number of segment cache
1162
DECLARE_Int32(segment_cache_capacity);
1163
DECLARE_Int32(segment_cache_fd_percentage);
1164
DECLARE_Int32(segment_cache_memory_percentage);
1165
DECLARE_mInt32(estimated_mem_per_column_reader);
1166
1167
// enable binlog
1168
DECLARE_Bool(enable_feature_binlog);
1169
1170
// enable set in BitmapValue
1171
DECLARE_Bool(enable_set_in_bitmap_value);
1172
1173
// max number of hdfs file handle in cache
1174
DECLARE_Int64(max_hdfs_file_handle_cache_num);
1175
DECLARE_Int32(max_hdfs_file_handle_cache_time_sec);
1176
1177
// max number of meta info of external files, such as parquet footer
1178
DECLARE_Int64(max_external_file_meta_cache_num);
1179
// Apply delete pred in cumu compaction
1180
DECLARE_mBool(enable_delete_when_cumu_compaction);
1181
1182
// max_write_buffer_number for rocksdb
1183
DECLARE_Int32(rocksdb_max_write_buffer_number);
1184
1185
// Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false.
1186
DECLARE_mBool(allow_zero_date);
1187
// Allow invalid decimalv2 literal for compatible with old version. Recommend set it false strongly.
1188
DECLARE_mBool(allow_invalid_decimalv2_literal);
1189
// Allow to specify kerberos credentials cache path.
1190
DECLARE_mString(kerberos_ccache_path);
1191
// set krb5.conf path, use "/etc/krb5.conf" by default
1192
DECLARE_mString(kerberos_krb5_conf_path);
1193
1194
// Values include `none`, `glog`, `boost`, `glibc`, `libunwind`
1195
DECLARE_mString(get_stack_trace_tool);
1196
DECLARE_mBool(enable_address_sanitizers_with_stack_trace);
1197
1198
// DISABLED: Don't resolve location info.
1199
// FAST: Perform CU lookup using .debug_aranges (might be incomplete).
1200
// FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure.
1201
// FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL.
1202
DECLARE_mString(dwarf_location_info_mode);
1203
1204
// the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
1205
DECLARE_mInt64(auto_inc_prefetch_size_ratio);
1206
1207
// the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer
1208
DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
1209
1210
// number of threads that fetch auto-inc ranges from FE
1211
DECLARE_mInt64(auto_inc_fetch_thread_num);
1212
// Max connection cache num for point lookup queries
1213
DECLARE_mInt64(lookup_connection_cache_bytes_limit);
1214
1215
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
1216
DECLARE_mInt64(LZ4_HC_compression_level);
1217
// Whether flatten nested arrays in variant column
1218
// Notice: TEST ONLY
1219
DECLARE_mBool(variant_enable_flatten_nested);
1220
// Threshold of a column as sparse column
1221
// Notice: TEST ONLY
1222
DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column);
1223
// Threshold to estimate a column is sparsed
1224
// Notice: TEST ONLY
1225
DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column);
1226
// Treat invalid json format str as string, instead of throwing exception if false
1227
DECLARE_mBool(variant_throw_exeception_on_invalid_json);
1228
1229
DECLARE_mBool(enable_merge_on_write_correctness_check);
1230
// rowid conversion correctness check when compaction for mow table
1231
DECLARE_mBool(enable_rowid_conversion_correctness_check);
1232
// missing rows correctness check when compaction for mow table
1233
DECLARE_mBool(enable_missing_rows_correctness_check);
1234
// When the number of missing versions is more than this value, do not directly
1235
// retry the publish and handle it through async publish.
1236
DECLARE_mInt32(mow_publish_max_discontinuous_version_num);
1237
// When the version is not continuous for MOW table in publish phase and the gap between
1238
// current txn's publishing version and the max version of the tablet exceeds this value,
1239
// don't print warning log
1240
DECLARE_mInt32(publish_version_gap_logging_threshold);
1241
1242
// The secure path with user files, used in the `local` table function.
1243
DECLARE_mString(user_files_secure_path);
1244
1245
// If fe's frontend info has not been updated for more than fe_expire_duration_seconds, it will be regarded
1246
// as an abnormal fe, this will cause be to cancel this fe's related query.
1247
DECLARE_Int32(fe_expire_duration_seconds);
1248
1249
// If use stop_be.sh --grace, then BE has to wait all running queries to stop to avoiding running query failure
1250
// , but if the waiting time exceed the limit, then be will exit directly.
1251
// During this period, FE will not send any queries to BE and waiting for all running queries to stop.
1252
DECLARE_Int32(grace_shutdown_wait_seconds);
1253
1254
// BitmapValue serialize version.
1255
DECLARE_Int16(bitmap_serialize_version);
1256
1257
// group commit config
1258
DECLARE_String(group_commit_wal_path);
1259
DECLARE_Int32(group_commit_replay_wal_retry_num);
1260
DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds);
1261
DECLARE_Int32(group_commit_replay_wal_retry_interval_max_seconds);
1262
DECLARE_mInt32(group_commit_relay_wal_threads);
1263
// This config can be set to limit thread number in group commit request fragment thread pool.
1264
DECLARE_mInt32(group_commit_insert_threads);
1265
DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
1266
DECLARE_Bool(wait_internal_group_commit_finish);
1267
// Max size(bytes) of group commit queues, used for mem back pressure.
1268
DECLARE_mInt32(group_commit_queue_mem_limit);
1269
// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
1270
// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
1271
DECLARE_mString(group_commit_wal_max_disk_limit);
1272
DECLARE_Bool(group_commit_wait_replay_wal_finish);
1273
1274
// The configuration item is used to lower the priority of the scanner thread,
1275
// typically employed to ensure CPU scheduling for write operations.
1276
// Default is 0, which is default value of thread nice value, increase this value
1277
// to lower the priority of scan threads
1278
DECLARE_Int32(scan_thread_nice_value);
1279
// Used to modify the recycle interval of tablet schema cache
1280
DECLARE_mInt32(tablet_schema_cache_recycle_interval);
1281
// Granularity is at the column level
1282
DECLARE_mInt32(tablet_schema_cache_capacity);
1283
1284
// Use `LOG(FATAL)` to replace `throw` when true
1285
DECLARE_mBool(exit_on_exception);
1286
1287
// cgroup
1288
DECLARE_mString(doris_cgroup_cpu_path);
1289
1290
DECLARE_mBool(enable_workload_group_memory_gc);
1291
1292
// This config controls whether the s3 file writer would flush cache asynchronously
1293
DECLARE_Bool(enable_flush_file_cache_async);
1294
1295
// Remove predicate that is always true for a segment.
1296
DECLARE_Bool(ignore_always_true_predicate_for_segment);
1297
1298
// Ingest binlog work pool size
1299
DECLARE_Int32(ingest_binlog_work_pool_size);
1300
1301
// Download binlog rate limit, unit is KB/s
1302
DECLARE_Int32(download_binlog_rate_limit_kbs);
1303
1304
DECLARE_mInt32(buffered_reader_read_timeout_ms);
1305
1306
// whether to enable /api/snapshot api
1307
DECLARE_Bool(enable_snapshot_action);
1308
1309
// The max columns size for a tablet schema
1310
DECLARE_mInt32(variant_max_merged_tablet_schema_size);
1311
1312
DECLARE_mInt64(local_exchange_buffer_mem_limit);
1313
1314
DECLARE_mInt64(enable_debug_log_timeout_secs);
1315
1316
DECLARE_mBool(enable_column_type_check);
1317
1318
// Tolerance for the number of partition id 0 in rowset, default 0
1319
DECLARE_Int32(ignore_invalid_partition_id_rowset_num);
1320
1321
DECLARE_mInt32(report_query_statistics_interval_ms);
1322
DECLARE_mInt32(query_statistics_reserve_timeout_ms);
1323
1324
// consider two high usage disk at the same available level if they do not exceed this diff.
1325
DECLARE_mDouble(high_disk_avail_level_diff_usages);
1326
1327
// create tablet in partition random robin idx lru size, default 10000
1328
DECLARE_Int32(partition_disk_index_lru_size);
1329
DECLARE_String(spill_storage_root_path);
1330
// Spill storage limit specified as number of bytes
1331
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
1332
// or percentage of capaity ('<int>%').
1333
// Defaults to bytes if no unit is given.
1334
// Must larger than 0.
1335
// If specified as percentage, the final limit value is:
1336
//   disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit
1337
DECLARE_String(spill_storage_limit);
1338
DECLARE_mInt32(spill_gc_interval_ms);
1339
DECLARE_mInt32(spill_gc_work_time_ms);
1340
DECLARE_Int32(spill_io_thread_pool_thread_num);
1341
DECLARE_Int32(spill_io_thread_pool_queue_size);
1342
1343
DECLARE_mBool(check_segment_when_build_rowset_meta);
1344
1345
// max s3 client retry times
1346
DECLARE_mInt32(max_s3_client_retry);
1347
// When meet s3 429 error, the "get" request will
1348
// sleep s3_read_base_wait_time_ms (*1, *2, *3, *4) ms
1349
// get try again.
1350
// The max sleep time is s3_read_max_wait_time_ms
1351
// and the max retry time is max_s3_client_retry
1352
DECLARE_mInt32(s3_read_base_wait_time_ms);
1353
DECLARE_mInt32(s3_read_max_wait_time_ms);
1354
1355
// write as inverted index tmp directory
1356
DECLARE_String(tmp_file_dir);
1357
1358
// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client
1359
DECLARE_mString(ca_cert_file_paths);
1360
1361
/** Table sink configurations(currently contains only external table types) **/
1362
// Minimum data processed to scale writers in exchange when non partition writing
1363
DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold);
1364
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1365
DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold);
1366
// Minimum partition data processed to rebalance writers in exchange when partition writing
1367
DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold);
1368
// Maximum processed partition nums of per writer when partition writing
1369
DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer);
1370
1371
/** Hive sink configurations **/
1372
DECLARE_mInt64(hive_sink_max_file_size);
1373
1374
/** Iceberg sink configurations **/
1375
DECLARE_mInt64(iceberg_sink_max_file_size);
1376
1377
// Number of open tries, default 1 means only try to open once.
1378
// Retry the Open num_retries time waiting 100 milliseconds between retries.
1379
DECLARE_mInt32(thrift_client_open_num_tries);
1380
1381
DECLARE_mBool(ignore_schema_change_check);
1382
1383
//JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues.
1384
DECLARE_Bool(enable_jvm_monitor);
1385
1386
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
1387
DECLARE_mBool(skip_loading_stale_rowset_meta);
1388
// Whether to use file to record log. When starting BE with --console,
1389
// all logs will be written to both standard output and file.
1390
// Disable this option will no longer use file to record log.
1391
// Only works when starting BE with --console.
1392
DECLARE_Bool(enable_file_logger);
1393
// The time out milliseconds for remote fetch schema RPC
1394
DECLARE_mInt64(fetch_remote_schema_rpc_timeout_ms);
1395
1396
// The minimum row group size when exporting Parquet files.
1397
DECLARE_Int64(min_row_group_size);
1398
1399
DECLARE_mInt64(compaction_memory_bytes_limit);
1400
1401
DECLARE_mInt64(compaction_batch_size);
1402
1403
DECLARE_mBool(enable_parquet_page_index);
1404
1405
// Wheather to ignore not found file in external teble(eg, hive)
1406
// Default is true, if set to false, the not found file will result in query failure.
1407
DECLARE_mBool(ignore_not_found_file_in_external_table);
1408
1409
DECLARE_mInt64(tablet_meta_serialize_size_limit);
1410
1411
DECLARE_mInt64(pipeline_task_leakage_detect_period_secs);
1412
// To be compatible with hadoop's block compression
1413
DECLARE_mInt32(snappy_compression_block_size);
1414
DECLARE_mInt32(lz4_compression_block_size);
1415
1416
DECLARE_mBool(enable_pipeline_task_leakage_detect);
1417
1418
#ifdef BE_TEST
1419
// test s3
1420
DECLARE_String(test_s3_resource);
1421
DECLARE_String(test_s3_ak);
1422
DECLARE_String(test_s3_sk);
1423
DECLARE_String(test_s3_endpoint);
1424
DECLARE_String(test_s3_region);
1425
DECLARE_String(test_s3_bucket);
1426
DECLARE_String(test_s3_prefix);
1427
#endif
1428
1429
class Register {
1430
public:
1431
    struct Field {
1432
        const char* type = nullptr;
1433
        const char* name = nullptr;
1434
        void* storage = nullptr;
1435
        const char* defval = nullptr;
1436
        bool valmutable = false;
1437
        Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1438
              bool fvalmutable)
1439
                : type(ftype),
1440
                  name(fname),
1441
                  storage(fstorage),
1442
                  defval(fdefval),
1443
544
                  valmutable(fvalmutable) {}
1444
    };
1445
1446
public:
1447
    static std::map<std::string, Field>* _s_field_map;
1448
1449
public:
1450
    Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1451
540
             bool fvalmutable) {
1452
540
        if (_s_field_map == nullptr) {
1453
1
            _s_field_map = new std::map<std::string, Field>();
1454
1
        }
1455
540
        Field field(ftype, fname, fstorage, fdefval, fvalmutable);
1456
540
        _s_field_map->insert(std::make_pair(std::string(fname), field));
1457
540
    }
1458
};
1459
1460
// RegisterConfValidator class is used to store validator function of registered config fields in
1461
// Register::_s_field_map.
1462
// If any validator return false when BE bootstart, the bootstart will be terminated.
1463
// If validator return false when use http API to update some config, the config will not
1464
// be modified and the API will return failure.
1465
class RegisterConfValidator {
1466
public:
1467
    // Validator for each config name.
1468
    static std::map<std::string, std::function<bool()>>* _s_field_validator;
1469
1470
public:
1471
    RegisterConfValidator(const char* fname, const std::function<bool()>& validator) {
1472
        if (_s_field_validator == nullptr) {
1473
            _s_field_validator = new std::map<std::string, std::function<bool()>>();
1474
        }
1475
        // register validator to _s_field_validator
1476
        _s_field_validator->insert(std::make_pair(std::string(fname), validator));
1477
    }
1478
};
1479
1480
// configuration properties load from config file.
1481
class Properties {
1482
public:
1483
    // load conf from file, if must_exist is true and file does not exist, return false
1484
    bool load(const char* conf_file, bool must_exist = true);
1485
1486
    // Find the config value by key from `file_conf_map`.
1487
    // If found, set `retval` to the config value,
1488
    // or set `retval` to `defstr`
1489
    // if retval is not set(in case defstr is nullptr), set is_retval_set to false
1490
    template <typename T>
1491
    bool get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set,
1492
                        std::string& rawval) const;
1493
1494
    void set(const std::string& key, const std::string& val);
1495
1496
    void set_force(const std::string& key, const std::string& val);
1497
1498
    // dump props to conf file
1499
    Status dump(const std::string& conffile);
1500
1501
private:
1502
    std::map<std::string, std::string> file_conf_map;
1503
};
1504
1505
// full configurations.
1506
extern std::map<std::string, std::string>* full_conf_map;
1507
1508
extern std::mutex custom_conf_lock;
1509
1510
// Init the config from `conf_file`.
1511
// If fill_conf_map is true, the updated config will also update the `full_conf_map`.
1512
// If must_exist is true and `conf_file` does not exist, this function will return false.
1513
// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
1514
bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist = true,
1515
          bool set_to_default = true);
1516
1517
Status set_config(const std::string& field, const std::string& value, bool need_persist = false,
1518
                  bool force = false);
1519
1520
Status persist_config(const std::string& field, const std::string& value);
1521
1522
std::mutex* get_mutable_string_config_lock();
1523
1524
std::vector<std::vector<std::string>> get_config_info();
1525
1526
Status set_fuzzy_configs();
1527
1528
void update_config(const std::string& field, const std::string& value);
1529
1530
} // namespace config
1531
} // namespace doris