Coverage Report

Created: 2025-04-25 12:58

/root/doris/be/src/common/config.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <cstdint>
21
#include <functional>
22
#include <map>
23
#include <mutex>
24
#include <string>
25
#include <utility>
26
#include <vector>
27
28
#define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME
29
30
#define DECLARE_Bool(name) DECLARE_FIELD(bool, name)
31
#define DECLARE_Int16(name) DECLARE_FIELD(int16_t, name)
32
#define DECLARE_Int32(name) DECLARE_FIELD(int32_t, name)
33
#define DECLARE_Int64(name) DECLARE_FIELD(int64_t, name)
34
#define DECLARE_Double(name) DECLARE_FIELD(double, name)
35
#define DECLARE_String(name) DECLARE_FIELD(std::string, name)
36
#define DECLARE_Bools(name) DECLARE_FIELD(std::vector<bool>, name)
37
#define DECLARE_Int16s(name) DECLARE_FIELD(std::vector<int16_t>, name)
38
#define DECLARE_Int32s(name) DECLARE_FIELD(std::vector<int32_t>, name)
39
#define DECLARE_Int64s(name) DECLARE_FIELD(std::vector<int64_t>, name)
40
#define DECLARE_Doubles(name) DECLARE_FIELD(std::vector<double>, name)
41
#define DECLARE_Strings(name) DECLARE_FIELD(std::vector<std::string>, name)
42
#define DECLARE_mBool(name) DECLARE_FIELD(bool, name)
43
#define DECLARE_mInt16(name) DECLARE_FIELD(int16_t, name)
44
#define DECLARE_mInt32(name) DECLARE_FIELD(int32_t, name)
45
#define DECLARE_mInt64(name) DECLARE_FIELD(int64_t, name)
46
#define DECLARE_mDouble(name) DECLARE_FIELD(double, name)
47
#define DECLARE_mString(name) DECLARE_FIELD(std::string, name)
48
49
#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE)                      \
50
21
    FIELD_TYPE FIELD_NAME;                                                                   \
51
21
    static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &(FIELD_NAME), FIELD_DEFAULT, \
52
21
                                     VALMUTABLE);
53
54
#define DEFINE_VALIDATOR(FIELD_NAME, VALIDATOR)              \
55
    static auto validator_##FIELD_NAME = VALIDATOR;          \
56
    static RegisterConfValidator reg_validator_##FIELD_NAME( \
57
            #FIELD_NAME, []() -> bool { return validator_##FIELD_NAME(FIELD_NAME); });
58
59
1
#define DEFINE_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false)
60
1
#define DEFINE_Bools(name, defaultstr) DEFINE_FIELD(std::vector<bool>, name, defaultstr, false)
61
1
#define DEFINE_Doubles(name, defaultstr) DEFINE_FIELD(std::vector<double>, name, defaultstr, false)
62
1
#define DEFINE_Int16s(name, defaultstr) DEFINE_FIELD(std::vector<int16_t>, name, defaultstr, false)
63
1
#define DEFINE_Int32s(name, defaultstr) DEFINE_FIELD(std::vector<int32_t>, name, defaultstr, false)
64
1
#define DEFINE_Int64s(name, defaultstr) DEFINE_FIELD(std::vector<int64_t>, name, defaultstr, false)
65
3
#define DEFINE_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false)
66
1
#define DEFINE_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false)
67
1
#define DEFINE_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false)
68
1
#define DEFINE_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false)
69
2
#define DEFINE_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false)
70
#define DEFINE_Strings(name, defaultstr) \
71
1
    DEFINE_FIELD(std::vector<std::string>, name, defaultstr, false)
72
1
#define DEFINE_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true)
73
1
#define DEFINE_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true)
74
1
#define DEFINE_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true)
75
1
#define DEFINE_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true)
76
1
#define DEFINE_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true)
77
1
#define DEFINE_mString(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, true)
78
#define DEFINE_Validator(name, validator) DEFINE_VALIDATOR(name, validator)
79
80
namespace doris {
81
class Status;
82
83
// If you want to modify the value of config, please go to common/config.cpp to modify.
84
namespace config {
85
// Dir of custom config file
86
DECLARE_String(custom_config_dir);
87
88
// Dir of jdbc drivers
89
DECLARE_String(jdbc_drivers_dir);
90
91
// cluster id
92
DECLARE_Int32(cluster_id);
93
// port on which BackendService is exported
94
DECLARE_Int32(be_port);
95
96
// port for brpc
97
DECLARE_Int32(brpc_port);
98
99
// port for arrow flight sql
100
// Default -1, do not start arrow flight sql server.
101
DECLARE_Int32(arrow_flight_sql_port);
102
103
// If the external client cannot directly access priority_networks, set public_host to be accessible
104
// to external client.
105
// There are usually two usage scenarios:
106
// 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network.
107
// However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be
108
// randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx.
109
// 2. if priority_networks is an internal network IP, and BE node has its own independent external IP,
110
// but Doris currently does not support modifying priority_networks, setting public_host to the real external IP.
111
DECLARE_mString(public_host);
112
113
// If the BE node is connected to the external network through a reverse proxy like Nginx
114
// and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy
115
// `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example:
116
// upstream arrowflight {
117
//    server 10.16.10.8:8069;
118
//    server 10.16.10.8:8068;
119
//}
120
// server {
121
//    listen 8167 http2;
122
//    listen [::]:8167 http2;
123
//    server_name doris.arrowflight.com;
124
// }
125
DECLARE_Int32(arrow_flight_sql_proxy_port);
126
127
// the number of bthreads for brpc, the default value is set to -1,
128
// which means the number of bthreads is #cpu-cores
129
DECLARE_Int32(brpc_num_threads);
130
DECLARE_Int32(brpc_idle_timeout_sec);
131
132
// Declare a selection strategy for those servers have many ips.
133
// Note that there should at most one ip match this list.
134
// This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
135
// If no ip match this rule, will choose one randomly.
136
DECLARE_String(priority_networks);
137
138
// performance moderate or compact, only tcmalloc compile
139
DECLARE_String(memory_mode);
140
141
// if true, process memory limit and memory usage based on cgroup memory info.
142
DECLARE_mBool(enable_use_cgroup_memory_info);
143
144
// process memory limit specified as number of bytes
145
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
146
// or percentage of the physical memory ('<int>%').
147
// defaults to bytes if no unit is given"
148
// must larger than 0. and if larger than physical memory size,
149
// it will be set to physical memory size.
150
DECLARE_String(mem_limit);
151
152
// Soft memory limit as a fraction of hard memory limit.
153
DECLARE_Double(soft_mem_limit_frac);
154
155
// Cache capacity reduce mem limit as a fraction of soft mem limit.
156
DECLARE_mDouble(cache_capacity_reduce_mem_limit_frac);
157
158
// Schema change memory limit as a fraction of soft memory limit.
159
DECLARE_Double(schema_change_mem_limit_frac);
160
161
// Many modern allocators (for example) do not do a mremap for
162
// realloc, even in case of large enough chunks of memory. Although this allows
163
// you to increase performance and reduce memory consumption during realloc.
164
// To fix this, we do mremap manually if the chunk of memory is large enough.
165
//
166
// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
167
// space is very slow, especially in the case of a large number of threads. We
168
// expect that the set of operations mmap/something to do/mremap can only be
169
// performed about 1000 times per second.
170
//
171
// P.S. This is also required, because tcmalloc can not allocate a chunk of
172
// memory greater than 16 GB.
173
DECLARE_mInt64(mmap_threshold); // bytes
174
175
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
176
// Increase can reduce the number of hash table resize, but may waste more memory.
177
DECLARE_mInt32(hash_table_double_grow_degree);
178
179
// The max fill rate for hash table
180
DECLARE_mInt32(max_fill_rate);
181
182
DECLARE_mInt32(double_resize_threshold);
183
184
// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default -1.
185
// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%), which is 3.2G on a 64G machine.
186
// Turn up max. more memory buffers will be reserved for Memory GC.
187
// Turn down max. will use as much memory as possible.
188
// note that: `max_` prefix should be removed, but keep it for compatibility.
189
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
190
191
// reserve a small amount of memory so we do not trigger MinorGC
192
DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
193
194
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
195
DECLARE_mString(process_minor_gc_size);
196
DECLARE_mString(process_full_gc_size);
197
198
// If true, when the process does not exceed the soft mem limit, the query memory will not be limited;
199
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
200
// used memory and the exec_mem_limit will be canceled.
201
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
202
DECLARE_mBool(enable_query_memory_overcommit);
203
204
// gc will release cache, cancel task, and task will wait for gc to release memory,
205
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
206
DECLARE_mBool(disable_memory_gc);
207
208
// if false, turn off all stacktrace
209
DECLARE_mBool(enable_stacktrace);
210
211
// when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G,
212
// if alloc successful, will print a warning with stacktrace, but not prevent memory alloc.
213
// if alloc failed using Doris Allocator, will print stacktrace in error log.
214
// if is -1, disable print stacktrace when alloc large memory.
215
DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
216
217
// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled.
218
// if you need a core dump to analyze large memory allocation,
219
// modify this parameter to crash when large memory allocation occur will help
220
DECLARE_mInt64(crash_in_alloc_large_memory_bytes);
221
222
// The actual meaning of this parameter is `debug_memory`.
223
// 1. crash in memory tracker inaccurate, if memory tracker value is inaccurate, BE will crash.
224
//    usually used in test environments, default value is false.
225
// 2. print more memory logs.
226
DECLARE_mBool(crash_in_memory_tracker_inaccurate);
227
228
// default is true. if any memory tracking in Orphan mem tracker will report error.
229
// !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time.
230
// allocator free memory not need to check, because when the thread memory tracker label is Orphan,
231
// use the tracker saved in Allocator.
232
DECLARE_mBool(enable_memory_orphan_check);
233
234
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
235
DECLARE_mInt32(thread_wait_gc_max_milliseconds);
236
237
// reach mem limit, don't serialize in batch
238
DECLARE_mInt64(pre_serialize_keys_limit_bytes);
239
240
// the port heartbeat service used
241
DECLARE_Int32(heartbeat_service_port);
242
// the count of heart beat service
243
DECLARE_Int32(heartbeat_service_thread_count);
244
// the count of thread to create table
245
DECLARE_Int32(create_tablet_worker_count);
246
// the count of thread to drop table
247
DECLARE_Int32(drop_tablet_worker_count);
248
// the count of thread to batch load
249
DECLARE_Int32(push_worker_count_normal_priority);
250
// the count of thread to high priority batch load
251
DECLARE_Int32(push_worker_count_high_priority);
252
// the count of thread to publish version
253
DECLARE_Int32(publish_version_worker_count);
254
// the count of tablet thread to publish version
255
DECLARE_Int32(tablet_publish_txn_max_thread);
256
// the timeout of EnginPublishVersionTask
257
DECLARE_Int32(publish_version_task_timeout_s);
258
// the count of thread to calc delete bitmap
259
DECLARE_Int32(calc_delete_bitmap_max_thread);
260
// the count of thread to calc delete bitmap worker, only used for cloud
261
DECLARE_Int32(calc_delete_bitmap_worker_count);
262
// the count of thread to calc tablet delete bitmap task, only used for cloud
263
DECLARE_Int32(calc_tablet_delete_bitmap_task_max_thread);
264
// the count of thread to clear transaction task
265
DECLARE_Int32(clear_transaction_task_worker_count);
266
// the count of thread to delete
267
DECLARE_Int32(delete_worker_count);
268
// the count of thread to alter table
269
DECLARE_Int32(alter_tablet_worker_count);
270
// the count of thread to alter index
271
DECLARE_Int32(alter_index_worker_count);
272
// the count of thread to clone
273
DECLARE_Int32(clone_worker_count);
274
// the count of thread to clone
275
DECLARE_Int32(storage_medium_migrate_count);
276
// the count of thread to check consistency
277
DECLARE_Int32(check_consistency_worker_count);
278
// the count of thread to upload
279
DECLARE_Int32(upload_worker_count);
280
// the count of thread to download
281
DECLARE_Int32(download_worker_count);
282
// the count of thread to make snapshot
283
DECLARE_Int32(make_snapshot_worker_count);
284
// the count of thread to release snapshot
285
DECLARE_Int32(release_snapshot_worker_count);
286
// report random wait a little time to avoid FE receiving multiple be reports at the same time.
287
// do not set it to false for production environment
288
DECLARE_mBool(report_random_wait);
289
// the interval time(seconds) for agent report tasks signature to FE
290
DECLARE_mInt32(report_task_interval_seconds);
291
// the interval time(seconds) for refresh storage policy from FE
292
DECLARE_mInt32(storage_refresh_storage_policy_task_interval_seconds);
293
// the interval time(seconds) for agent report disk state to FE
294
DECLARE_mInt32(report_disk_state_interval_seconds);
295
// the interval time(seconds) for agent report olap table to FE
296
DECLARE_mInt32(report_tablet_interval_seconds);
297
// the max download speed(KB/s)
298
DECLARE_mInt32(max_download_speed_kbps);
299
// download low speed limit(KB/s)
300
DECLARE_mInt32(download_low_speed_limit_kbps);
301
// download low speed time(seconds)
302
DECLARE_mInt32(download_low_speed_time);
303
// whether to download small files in batch.
304
DECLARE_mBool(enable_batch_download);
305
// whether to check md5sum when download
306
DECLARE_mBool(enable_download_md5sum_check);
307
// download binlog meta timeout
308
DECLARE_mInt32(download_binlog_meta_timeout_ms);
309
310
// deprecated, use env var LOG_DIR in be.conf
311
DECLARE_String(sys_log_dir);
312
// for udf
313
DECLARE_String(user_function_dir);
314
// INFO, WARNING, ERROR, FATAL
315
DECLARE_String(sys_log_level);
316
// TIME-DAY, TIME-HOUR, SIZE-MB-nnn
317
DECLARE_String(sys_log_roll_mode);
318
// log roll num
319
DECLARE_Int32(sys_log_roll_num);
320
// verbose log
321
DECLARE_Strings(sys_log_verbose_modules);
322
// verbose log level
323
DECLARE_Int32(sys_log_verbose_level);
324
// verbose log FLAGS_v
325
DECLARE_Int32(sys_log_verbose_flags_v);
326
// log buffer level
327
DECLARE_String(log_buffer_level);
328
// log enable custom date time format
329
DECLARE_Bool(sys_log_enable_custom_date_time_format);
330
// log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time)
331
DECLARE_String(sys_log_custom_date_time_format);
332
// log custom date time milliseconds format (fmt::format)
333
DECLARE_String(sys_log_custom_date_time_ms_format);
334
335
// number of threads available to serve backend execution requests
336
DECLARE_Int32(be_service_threads);
337
338
// interval between profile reports; in seconds
339
DECLARE_mInt32(status_report_interval);
340
DECLARE_mInt32(pipeline_status_report_interval);
341
// if true, each disk will have a separate thread pool for scanner
342
DECLARE_Bool(doris_enable_scanner_thread_pool_per_disk);
343
// the timeout of a work thread to wait the blocking priority queue to get a task
344
DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
345
// number of scanner thread pool size for olap table
346
// and the min thread num of remote scanner thread pool
347
DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
348
DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num);
349
// number of batch size to fetch the remote split source
350
DECLARE_mInt32(remote_split_source_batch_size);
351
// max number of remote scanner thread pool size
352
// if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
353
DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
354
// number of olap scanner thread pool queue size
355
DECLARE_Int32(doris_scanner_thread_pool_queue_size);
356
// default thrift client connect timeout(in seconds)
357
DECLARE_mInt32(thrift_connect_timeout_seconds);
358
DECLARE_mInt32(fetch_rpc_timeout_seconds);
359
// default thrift client retry interval (in milliseconds)
360
DECLARE_mInt64(thrift_client_retry_interval_ms);
361
// max message size of thrift request
362
// default: 100 * 1024 * 1024
363
DECLARE_mInt64(thrift_max_message_size);
364
// max row count number for single scan range, used in segmentv1
365
DECLARE_mInt32(doris_scan_range_row_count);
366
// max bytes number for single scan range, used in segmentv2
367
DECLARE_mInt32(doris_scan_range_max_mb);
368
// max bytes number for single scan block, used in segmentv2
369
DECLARE_mInt32(doris_scan_block_max_mb);
370
// single read execute fragment row number
371
DECLARE_mInt32(doris_scanner_row_num);
372
// single read execute fragment row bytes
373
DECLARE_mInt32(doris_scanner_row_bytes);
374
// single read execute fragment max run time millseconds
375
DECLARE_mInt32(doris_scanner_max_run_time_ms);
376
DECLARE_mInt32(min_bytes_in_scanner_queue);
377
// (Advanced) Maximum size of per-query receive-side buffer
378
DECLARE_mInt32(exchg_node_buffer_size_bytes);
379
DECLARE_mInt32(exchg_buffer_queue_capacity_factor);
380
381
DECLARE_mInt64(column_dictionary_key_ratio_threshold);
382
DECLARE_mInt64(column_dictionary_key_size_threshold);
383
// memory_limitation_per_thread_for_schema_change_bytes unit bytes
384
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
385
DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);
386
387
// all cache prune interval, used by GC and periodic thread.
388
DECLARE_mInt32(cache_prune_interval_sec);
389
DECLARE_mInt32(cache_periodic_prune_stale_sweep_sec);
390
// the clean interval of tablet lookup cache
391
DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec);
392
DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec);
393
DECLARE_mInt32(disk_stat_monitor_interval);
394
DECLARE_mInt32(unused_rowset_monitor_interval);
395
DECLARE_mInt32(quering_rowsets_evict_interval);
396
DECLARE_String(storage_root_path);
397
DECLARE_mString(broken_storage_path);
398
399
// Config is used to check incompatible old format hdr_ format
400
// whether doris uses strict way. When config is true, process will log fatal
401
// and exit. When config is false, process will only log warning.
402
DECLARE_Bool(storage_strict_check_incompatible_old_format);
403
404
// BE process will exit if the percentage of error disk reach this value.
405
DECLARE_mInt32(max_percentage_of_error_disk);
406
DECLARE_mInt32(default_num_rows_per_column_file_block);
407
// pending data policy
408
DECLARE_mInt32(pending_data_expire_time_sec);
409
// inc_rowset snapshot rs sweep time interval
410
DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec);
411
// tablet stale rowset sweep by threshold size
412
DECLARE_Bool(tablet_rowset_stale_sweep_by_size);
413
DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size);
414
// garbage sweep policy
415
DECLARE_Int32(max_garbage_sweep_interval);
416
DECLARE_Int32(min_garbage_sweep_interval);
417
// garbage sweep every batch will sleep 1ms
418
DECLARE_mInt32(garbage_sweep_batch_size);
419
DECLARE_mInt32(snapshot_expire_time_sec);
420
// It is only a recommended value. When the disk space is insufficient,
421
// the file storage period under trash dose not have to comply with this parameter.
422
DECLARE_mInt32(trash_file_expire_time_sec);
423
// minimum file descriptor number
424
// modify them upon necessity
425
DECLARE_Int32(min_file_descriptor_number);
426
DECLARE_mBool(disable_segment_cache);
427
DECLARE_String(row_cache_mem_limit);
428
429
// Cache for storage page size
430
DECLARE_String(storage_page_cache_limit);
431
// Shard size for page cache, the value must be power of two.
432
// It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions.
433
DECLARE_Int32(storage_page_cache_shard_size);
434
// Percentage for index page cache
435
// all storage page cache will be divided into data_page_cache and index_page_cache
436
DECLARE_Int32(index_page_cache_percentage);
437
// whether to disable page cache feature in storage
438
// TODO delete it. Divided into Data page, Index page, pk index page
439
DECLARE_Bool(disable_storage_page_cache);
440
// whether to disable row cache feature in storage
441
DECLARE_mBool(disable_storage_row_cache);
442
// whether to disable pk page cache feature in storage
443
DECLARE_Bool(disable_pk_storage_page_cache);
444
DECLARE_Bool(enable_non_pipeline);
445
446
// Cache for mow primary key storage page size, it's seperated from
447
// storage_page_cache_limit
448
DECLARE_String(pk_storage_page_cache_limit);
449
// data page size for primary key index
450
DECLARE_Int32(primary_key_data_page_size);
451
452
// inc_rowset snapshot rs sweep time interval
453
DECLARE_mInt32(data_page_cache_stale_sweep_time_sec);
454
DECLARE_mInt32(index_page_cache_stale_sweep_time_sec);
455
// great impact on the performance of MOW, so it can be longer.
456
DECLARE_mInt32(pk_index_page_cache_stale_sweep_time_sec);
457
458
DECLARE_Bool(enable_low_cardinality_optimize);
459
DECLARE_Bool(enable_low_cardinality_cache_code);
460
461
// be policy
462
// whether check compaction checksum
463
DECLARE_mBool(enable_compaction_checksum);
464
// whether disable automatic compaction task
465
DECLARE_mBool(disable_auto_compaction);
466
// whether enable vertical compaction
467
DECLARE_mBool(enable_vertical_compaction);
468
// whether enable ordered data compaction
469
DECLARE_mBool(enable_ordered_data_compaction);
470
// In vertical compaction, column number for every group
471
DECLARE_mInt32(vertical_compaction_num_columns_per_group);
472
// In vertical compaction, max memory usage for row_source_buffer
473
DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
474
// In vertical compaction, max dest segment file size
475
DECLARE_mInt64(vertical_compaction_max_segment_size);
476
477
// If enabled, segments will be flushed column by column
478
DECLARE_mBool(enable_vertical_segment_writer);
479
480
// In ordered data compaction, min segment size for input rowset
481
DECLARE_mInt32(ordered_data_compaction_min_segment_size);
482
483
// This config can be set to limit thread number in compaction thread pool.
484
DECLARE_mInt32(max_base_compaction_threads);
485
DECLARE_mInt32(max_cumu_compaction_threads);
486
DECLARE_mInt32(max_single_replica_compaction_threads);
487
488
DECLARE_Bool(enable_base_compaction_idle_sched);
489
DECLARE_mInt64(base_compaction_min_rowset_num);
490
DECLARE_mInt64(base_compaction_max_compaction_score);
491
DECLARE_mDouble(base_compaction_min_data_ratio);
492
DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes);
493
494
DECLARE_Bool(enable_skip_tablet_compaction);
495
DECLARE_mInt32(skip_tablet_compaction_second);
496
// output rowset of cumulative compaction total disk size exceed this config size,
497
// this rowset will be given to base compaction, unit is m byte.
498
DECLARE_mInt64(compaction_promotion_size_mbytes);
499
500
// output rowset of cumulative compaction total disk size exceed this config ratio of
501
// base rowset's total disk size, this rowset will be given to base compaction. The value must be between
502
// 0 and 1.
503
DECLARE_mDouble(compaction_promotion_ratio);
504
505
// the smallest size of rowset promotion. When the rowset is less than this config, this
506
// rowset will be not given to base compaction. The unit is m byte.
507
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
508
509
// When output rowset of cumulative compaction total version count (end_version - start_version)
510
// exceed this config count, the rowset will be moved to base compaction
511
// NOTE: this config will work for unique key merge-on-write table only, to reduce version count
512
// related cost on delete bitmap more effectively.
513
DECLARE_mInt64(compaction_promotion_version_count);
514
515
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
516
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
517
DECLARE_mInt64(compaction_min_size_mbytes);
518
519
// cumulative compaction policy: min and max delta file's number
520
DECLARE_mInt64(cumulative_compaction_min_deltas);
521
DECLARE_mInt64(cumulative_compaction_max_deltas);
522
DECLARE_mInt32(cumulative_compaction_max_deltas_factor);
523
524
// This config can be set to limit thread number in  multiget thread pool.
525
DECLARE_mInt32(multi_get_max_threads);
526
527
// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
528
DECLARE_mInt64(total_permits_for_compaction_score);
529
530
// sleep interval in ms after generated compaction tasks
531
DECLARE_mInt32(generate_compaction_tasks_interval_ms);
532
// sleep interval in second after update replica infos
533
DECLARE_mInt32(update_replica_infos_interval_seconds);
534
535
// Compaction task number per disk.
536
// Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each.
537
DECLARE_mInt32(compaction_task_num_per_disk);
538
// compaction thread num for fast disk(typically .SSD), must be greater than 2.
539
DECLARE_mInt32(compaction_task_num_per_fast_disk);
540
541
// How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation.
542
DECLARE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round);
543
544
// Minimum number of threads required in the thread pool to activate the large cumu compaction delay strategy.
545
// The delay strategy is only applied when the thread pool has at least this many threads.
546
DECLARE_mInt32(large_cumu_compaction_task_min_thread_num);
547
// Maximum size threshold (in bytes) for input rowsets. Compaction tasks with input size
548
// exceeding this threshold will be delayed when thread pool is near capacity. Default 100MB.
549
DECLARE_mInt32(large_cumu_compaction_task_bytes_threshold);
550
// Maximum row count threshold for compaction input. Compaction tasks with row count
551
// exceeding this threshold will be delayed when thread pool is near capacity. Default 1 million.
552
DECLARE_mInt32(large_cumu_compaction_task_row_num_threshold);
553
554
// Not compact the invisible versions, but with some limitations:
555
// if not timeout, keep no more than compaction_keep_invisible_version_max_count versions;
556
// if timeout, keep no more than compaction_keep_invisible_version_min_count versions.
557
DECLARE_mInt32(compaction_keep_invisible_version_timeout_sec);
558
DECLARE_mInt32(compaction_keep_invisible_version_min_count);
559
DECLARE_mInt32(compaction_keep_invisible_version_max_count);
560
561
// Threshold to logging compaction trace, in seconds.
562
DECLARE_mInt32(base_compaction_trace_threshold);
563
DECLARE_mInt32(cumulative_compaction_trace_threshold);
564
DECLARE_mBool(disable_compaction_trace_log);
565
566
// Interval to picking rowset to compact, in seconds
567
DECLARE_mInt64(pick_rowset_to_compact_interval_sec);
568
569
// Compaction priority schedule
570
DECLARE_mBool(enable_compaction_priority_scheduling);
571
DECLARE_mInt32(low_priority_compaction_task_num_per_disk);
572
DECLARE_mInt32(low_priority_compaction_score_threshold);
573
574
// Thread count to do tablet meta checkpoint, -1 means use the data directories count.
575
DECLARE_Int32(max_meta_checkpoint_threads);
576
577
// Threshold to logging agent task trace, in seconds.
578
DECLARE_mInt32(agent_task_trace_threshold_sec);
579
580
// This config can be set to limit thread number in tablet migration thread pool.
581
DECLARE_Int32(min_tablet_migration_threads);
582
DECLARE_Int32(max_tablet_migration_threads);
583
584
DECLARE_mInt32(finished_migration_tasks_size);
585
// If size less than this, the remaining rowsets will be force to complete
586
DECLARE_mInt32(migration_remaining_size_threshold_mb);
587
// If the task runs longer than this time, the task will be terminated, in seconds.
588
// timeout = std::max(migration_task_timeout_secs,  tablet size / 1MB/s)
589
DECLARE_mInt32(migration_task_timeout_secs);
590
// timeout for try_lock migration lock
591
DECLARE_Int64(migration_lock_timeout_ms);
592
593
// Port to start debug webserver on
594
DECLARE_Int32(webserver_port);
595
// Https enable flag
596
DECLARE_Bool(enable_https);
597
// Path of certificate
598
DECLARE_String(ssl_certificate_path);
599
// Path of private key
600
DECLARE_String(ssl_private_key_path);
601
// Whether to check authorization
602
DECLARE_Bool(enable_all_http_auth);
603
// Number of webserver workers
604
DECLARE_Int32(webserver_num_workers);
605
606
DECLARE_Bool(enable_single_replica_load);
607
// Number of download workers for single replica load
608
DECLARE_Int32(single_replica_load_download_num_workers);
609
610
// Used for mini Load. mini load data file will be removed after this time.
611
DECLARE_Int64(load_data_reserve_hours);
612
// log error log will be removed after this time
613
DECLARE_mInt64(load_error_log_reserve_hours);
614
// error log size limit, default 200MB
615
DECLARE_mInt64(load_error_log_limit_bytes);
616
617
// be brpc interface is classified into two categories: light and heavy
618
// each category has diffrent thread number
619
// threads to handle heavy api interface, such as transmit_data/transmit_block etc
620
DECLARE_Int32(brpc_heavy_work_pool_threads);
621
// threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start
622
DECLARE_Int32(brpc_light_work_pool_threads);
623
DECLARE_Int32(brpc_heavy_work_pool_max_queue_size);
624
DECLARE_Int32(brpc_light_work_pool_max_queue_size);
625
DECLARE_mBool(enable_bthread_transmit_block);
626
DECLARE_Int32(brpc_arrow_flight_work_pool_threads);
627
DECLARE_Int32(brpc_arrow_flight_work_pool_max_queue_size);
628
629
// The maximum amount of data that can be processed by a stream load
630
DECLARE_mInt64(streaming_load_max_mb);
631
// Some data formats, such as JSON, cannot be streamed.
632
// Therefore, it is necessary to limit the maximum number of
633
// such data when using stream load to prevent excessive memory consumption.
634
DECLARE_mInt64(streaming_load_json_max_mb);
635
// the alive time of a TabletsChannel.
636
// If the channel does not receive any data till this time,
637
// the channel will be removed.
638
DECLARE_mInt32(streaming_load_rpc_max_alive_time_sec);
639
// the timeout of a rpc to open the tablet writer in remote BE.
640
// short operation time, can set a short timeout
641
DECLARE_Int32(tablet_writer_open_rpc_timeout_sec);
642
// You can ignore brpc error '[E1011]The server is overcrowded' when writing data.
643
DECLARE_mBool(tablet_writer_ignore_eovercrowded);
644
DECLARE_mInt32(slave_replica_writer_rpc_timeout_sec);
645
// Whether to enable stream load record function, the default is false.
646
// False: disable stream load record
647
DECLARE_mBool(enable_stream_load_record);
648
// batch size of stream load record reported to FE
649
DECLARE_mInt32(stream_load_record_batch_size);
650
// expire time of stream load record in rocksdb.
651
DECLARE_Int32(stream_load_record_expire_time_secs);
652
// time interval to clean expired stream load records
653
DECLARE_mInt64(clean_stream_load_record_interval_secs);
654
// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
655
DECLARE_mBool(enable_stream_load_commit_txn_on_be);
656
// The buffer size to store stream table function schema info
657
DECLARE_Int64(stream_tvf_buffer_size);
658
659
// OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc.
660
// You may need to lower the speed when the sink receiver bes are too busy.
661
DECLARE_mInt32(olap_table_sink_send_interval_microseconds);
662
// For auto partition, the send interval will multiply the factor
663
DECLARE_mDouble(olap_table_sink_send_interval_auto_partition_factor);
664
665
// Fragment thread pool
666
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_min);
667
DECLARE_Int32(fragment_mgr_asynic_work_pool_thread_num_max);
668
DECLARE_Int32(fragment_mgr_asynic_work_pool_queue_size);
669
670
// Control the number of disks on the machine.  If 0, this comes from the system settings.
671
DECLARE_Int32(num_disks);
672
// The maximum number of the threads per disk is also the max queue depth per disk.
673
DECLARE_Int32(num_threads_per_disk);
674
// The read size is the size of the reads sent to os.
675
// There is a trade off of latency and throughout, trying to keep disks busy but
676
// not introduce seeks.  The literature seems to agree that with 8 MB reads, random
677
// io and sequential io perform similarly.
678
DECLARE_Int32(read_size);       // 8 * 1024 * 1024, Read Size (in bytes)
679
DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes)
680
681
// for pprof
682
DECLARE_String(pprof_profile_dir);
683
// for jeprofile in jemalloc
684
DECLARE_mString(jeprofile_dir);
685
// Purge all unused dirty pages for all arenas.
686
DECLARE_mBool(enable_je_purge_dirty_pages);
687
// Purge all unused Jemalloc dirty pages for all arenas when exceed je_dirty_pages_mem_limit and process exceed soft limit.
688
DECLARE_mString(je_dirty_pages_mem_limit_percent);
689
690
// to forward compatibility, will be removed later
691
DECLARE_mBool(enable_token_check);
692
693
// to open/close system metrics
694
DECLARE_Bool(enable_system_metrics);
695
696
// Number of cores Doris will used, this will effect only when it's greater than 0.
697
// Otherwise, Doris will use all cores returned from "/proc/cpuinfo".
698
DECLARE_Int32(num_cores);
699
700
// When BE start, If there is a broken disk, BE process will exit by default.
701
// Otherwise, we will ignore the broken disk,
702
DECLARE_Bool(ignore_broken_disk);
703
704
// Sleep time in milliseconds between memory maintenance iterations
705
DECLARE_mInt32(memory_maintenance_sleep_time_ms);
706
707
// After full gc, no longer full gc and minor gc during sleep.
708
// After minor gc, no minor gc during sleep, but full gc is possible.
709
DECLARE_mInt32(memory_gc_sleep_time_ms);
710
711
// max write buffer size before flush, default 200MB
712
DECLARE_mInt64(write_buffer_size);
713
// max buffer size used in memtable for the aggregated table, default 400MB
714
DECLARE_mInt64(write_buffer_size_for_agg);
715
// max parallel flush task per memtable writer
716
DECLARE_mInt32(memtable_flush_running_count_limit);
717
718
DECLARE_Int32(load_process_max_memory_limit_percent); // 50%
719
720
// If the memory consumption of load jobs exceed load_process_max_memory_limit,
721
// all load jobs will hang there to wait for memtable flush. We should have a
722
// soft limit which can trigger the memtable flush for the load channel who
723
// consumes lagest memory size before we reach the hard limit. The soft limit
724
// might avoid all load jobs hang at the same time.
725
DECLARE_Int32(load_process_soft_mem_limit_percent);
726
727
// If load memory consumption is within load_process_safe_mem_permit_percent,
728
// memtable memory limiter will do nothing.
729
DECLARE_Int32(load_process_safe_mem_permit_percent);
730
731
// result buffer cancelled time (unit: second)
732
DECLARE_mInt32(result_buffer_cancelled_interval_time);
733
734
// arrow flight result sink buffer rows size, default 4096 * 8
735
DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows);
736
// The timeout for ADBC Client to wait for data using arrow flight reader.
737
// If the query is very complex and no result is generated after this time, consider increasing this timeout.
738
DECLARE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms);
739
740
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
741
DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency);
742
743
// sync tablet_meta when modifying meta
744
DECLARE_mBool(sync_tablet_meta);
745
746
// sync a file writer when it is closed
747
DECLARE_mBool(sync_file_on_close);
748
749
// default thrift rpc timeout ms
750
DECLARE_mInt32(thrift_rpc_timeout_ms);
751
752
// txn commit rpc timeout
753
DECLARE_mInt32(txn_commit_rpc_timeout_ms);
754
755
// If set to true, metric calculator will run
756
DECLARE_Bool(enable_metric_calculator);
757
758
// max consumer num in one data consumer group, for routine load
759
DECLARE_mInt32(max_consumer_num_per_group);
760
761
// the max size of thread pool for routine load task.
762
// this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5)
763
DECLARE_Int32(max_routine_load_thread_pool_size);
764
765
// max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
766
// default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached
767
DECLARE_mInt32(max_memory_sink_batch_count);
768
769
// This configuration is used for the context gc thread schedule period
770
// note: unit is minute, default is 5min
771
DECLARE_mInt32(scan_context_gc_interval_min);
772
773
// es scroll keep-alive
774
DECLARE_String(es_scroll_keepalive);
775
776
// HTTP connection timeout for es
777
DECLARE_mInt32(es_http_timeout_ms);
778
779
// the max client cache number per each host
780
// There are variety of client cache in BE, but currently we use the
781
// same cache size configuration.
782
// TODO(cmy): use different config to set different client cache if necessary.
783
DECLARE_Int32(max_client_cache_size_per_host);
784
785
DECLARE_Int32(max_master_fe_client_cache_size);
786
787
// Dir to save files downloaded by SmallFileMgr
788
DECLARE_String(small_file_dir);
789
// path gc
790
DECLARE_Bool(path_gc_check);
791
DECLARE_mInt32(path_gc_check_interval_second);
792
DECLARE_mInt32(path_gc_check_step);
793
DECLARE_mInt32(path_gc_check_step_interval_ms);
794
795
// The following 2 configs limit the max usage of disk capacity of a data dir.
796
// If both of these 2 threshold reached, no more data can be writen into that data dir.
797
// The percent of max used capacity of a data dir
798
DECLARE_mInt32(storage_flood_stage_usage_percent); // 90%
799
// The min bytes that should be left of a data dir
800
DECLARE_mInt64(storage_flood_stage_left_capacity_bytes); // 1GB
801
// number of thread for flushing memtable per store
802
DECLARE_Int32(flush_thread_num_per_store);
803
// number of thread for flushing memtable per store, for high priority load task
804
DECLARE_Int32(high_priority_flush_thread_num_per_store);
805
// number of threads = min(flush_thread_num_per_store * num_store,
806
//                         max_flush_thread_num_per_cpu * num_cpu)
807
DECLARE_Int32(max_flush_thread_num_per_cpu);
808
809
// workload group flush pool params
810
DECLARE_mInt32(wg_flush_thread_num_per_store);
811
DECLARE_mInt32(wg_flush_thread_num_per_cpu);
812
813
// config for tablet meta checkpoint
814
DECLARE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num);
815
DECLARE_mInt32(tablet_meta_checkpoint_min_interval_secs);
816
DECLARE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs);
817
818
// config for default rowset type
819
// Valid configs: ALPHA, BETA
820
DECLARE_String(default_rowset_type);
821
822
// Maximum size of a single message body in all protocols
823
DECLARE_Int64(brpc_max_body_size);
824
// Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED
825
// Default, if the physical memory is less than or equal to 64G, the value is 1G
826
//          if the physical memory is greater than 64G, the value is physical memory * mem_limit(0.8) / 1024 * 20
827
DECLARE_Int64(brpc_socket_max_unwritten_bytes);
828
// TODO(zxy): expect to be true in v1.3
829
// Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into
830
// Controller Attachment and send it through http brpc when the length of the Tuple/Block data
831
// is greater than 1.8G. This is to avoid the error of Request length overflow (2G).
832
DECLARE_mBool(transfer_large_data_by_brpc);
833
834
// max number of txns for every txn_partition_map in txn manager
835
// this is a self protection to avoid too many txns saving in manager
836
DECLARE_mInt64(max_runnings_transactions_per_txn_map);
837
838
// tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4
839
// this is a an enhancement for better performance to manage tablet
840
DECLARE_Int32(tablet_map_shard_size);
841
842
// txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4
843
// this is a an enhancement for better performance to manage txn
844
DECLARE_Int32(txn_map_shard_size);
845
846
// txn_lock shard size, the value is 2^n, n=0,1,2,3,4
847
// this is a an enhancement for better performance to commit and publish txn
848
DECLARE_Int32(txn_shard_size);
849
850
// Whether to continue to start be when load tablet from header failed.
851
DECLARE_Bool(ignore_load_tablet_failure);
852
853
// Whether to continue to start be when load tablet from header failed.
854
DECLARE_mBool(ignore_rowset_stale_unconsistent_delete);
855
856
// Set max cache's size of query results, the unit is M byte
857
DECLARE_Int32(query_cache_max_size_mb);
858
859
// Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb
860
DECLARE_Int32(query_cache_elasticity_size_mb);
861
862
// Maximum number of cache partitions corresponding to a SQL
863
DECLARE_Int32(query_cache_max_partition_count);
864
865
// Maximum number of version of a tablet. If the version num of a tablet exceed limit,
866
// the load process will reject new incoming load job of this tablet.
867
// This is to avoid too many version num.
868
DECLARE_mInt32(max_tablet_version_num);
869
870
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
871
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
872
DECLARE_String(thrift_server_type_of_fe);
873
874
// disable zone map index when page row is too few
875
DECLARE_mInt32(zone_map_row_num_threshold);
876
877
// aws sdk log level
878
//    Off = 0,
879
//    Fatal = 1,
880
//    Error = 2,
881
//    Warn = 3,
882
//    Info = 4,
883
//    Debug = 5,
884
//    Trace = 6
885
DECLARE_Int32(aws_log_level);
886
887
// the buffer size when read data from remote storage like s3
888
DECLARE_mInt32(remote_storage_read_buffer_mb);
889
890
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
891
// smaller than this value will continue to accumulate. specified as number of bytes.
892
// Decreasing this value will increase the frequency of consume/release.
893
// Increasing this value will cause MemTracker statistics to be inaccurate.
894
DECLARE_mInt32(mem_tracker_consume_min_size_bytes);
895
896
// The version information of the tablet will be stored in the memory
897
// in an adjacency graph data structure.
898
// And as the new version is written and the old version is deleted,
899
// the data structure will begin to have empty vertex with no edge associations(orphan vertex).
900
// This config is used to control that when the proportion of orphan vertex is greater than the threshold,
901
// the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely.
902
// This config usually only needs to be modified during testing.
903
// In most cases, it does not need to be modified.
904
DECLARE_mDouble(tablet_version_graph_orphan_vertex_ratio);
905
906
// share delta writers when memtable_on_sink_node = true
907
DECLARE_Bool(share_delta_writers);
908
// timeout for open load stream rpc in ms
909
DECLARE_Int64(open_load_stream_timeout_ms);
910
// enable write background when using brpc stream
911
DECLARE_mBool(enable_brpc_stream_write_background);
912
913
// brpc streaming max_buf_size in bytes
914
DECLARE_Int64(load_stream_max_buf_size);
915
// brpc streaming messages_in_batch
916
DECLARE_Int32(load_stream_messages_in_batch);
917
// brpc streaming StreamWait seconds on EAGAIN
918
DECLARE_Int32(load_stream_eagain_wait_seconds);
919
// max tasks per flush token in load stream
920
DECLARE_Int32(load_stream_flush_token_max_tasks);
921
// max wait flush token time in load stream
922
DECLARE_Int32(load_stream_max_wait_flush_token_time_ms);
923
// number of send batch thread pool size
924
DECLARE_Int32(send_batch_thread_pool_thread_num);
925
// number of send batch thread pool queue size
926
DECLARE_Int32(send_batch_thread_pool_queue_size);
927
928
// Limit the number of segment of a newly created rowset.
929
// The newly created rowset may to be compacted after loading,
930
// so if there are too many segment in a rowset, the compaction process
931
// will run out of memory.
932
// When doing compaction, each segment may take at least 1MB buffer.
933
DECLARE_mInt32(max_segment_num_per_rowset);
934
935
// Store segment without compression if a segment is smaller than
936
// segment_compression_threshold_kb.
937
DECLARE_mInt32(segment_compression_threshold_kb);
938
939
// The connection timeout when connecting to external table such as odbc table.
940
DECLARE_mInt32(external_table_connect_timeout_sec);
941
942
// Time to clean up useless JDBC connection pool cache
943
DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
944
945
// Global bitmap cache capacity for aggregation cache, size in bytes
946
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
947
DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
948
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
949
950
// A common object cache depends on an Sharded LRU Cache.
951
DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
952
953
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
954
// If the dependent kafka broker version older than 0.10.0.0,
955
// the value of kafka_api_version_request should be false, and the
956
// value set by the fallback version kafka_broker_version_fallback will be used,
957
// and the valid values are: 0.9.0.x, 0.8.x.y.
958
DECLARE_String(kafka_api_version_request);
959
DECLARE_String(kafka_broker_version_fallback);
960
DECLARE_mString(kafka_debug);
961
962
// The number of pool siz of routine load consumer.
963
// If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608
964
// Change this size to 0 to fix it temporarily.
965
DECLARE_mInt32(routine_load_consumer_pool_size);
966
967
// Used in single-stream-multi-table load. When receive a batch of messages from kafka,
968
// if the size of batch is more than this threshold, we will request plans for all related tables.
969
DECLARE_Int32(multi_table_batch_plan_threshold);
970
971
// Used in single-stream-multi-table load. When receiving a batch of messages from Kafka,
972
// if the size of the table wait for plan is more than this threshold, we will request plans for all related tables.
973
// The param is aimed to avoid requesting and executing too many plans at once.
974
// Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC
975
// and improve the real-time processing of data.
976
DECLARE_Int32(multi_table_max_wait_tables);
977
978
// When the timeout of a load task is less than this threshold,
979
// Doris treats it as a high priority task.
980
// high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic.
981
// this threshold is mainly used to identify routine load tasks and should not be modified if not necessary.
982
DECLARE_mInt32(load_task_high_priority_threshold_second);
983
984
// The min timeout of load rpc (add batch, close)
985
// Because a load rpc may be blocked for a while.
986
// Increase this config may avoid rpc timeout.
987
DECLARE_mInt32(min_load_rpc_timeout_ms);
988
989
// use which protocol to access function service, candicate is baidu_std/h2:grpc
990
DECLARE_String(function_service_protocol);
991
992
// use which load balancer to select server to connect
993
DECLARE_String(rpc_load_balancer);
994
995
// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
996
// so we set a soft limit, default is 1MB
997
DECLARE_mInt32(string_type_length_soft_limit_bytes);
998
999
DECLARE_mInt32(jsonb_type_length_soft_limit_bytes);
1000
1001
// Threshold fo reading a small file into memory
1002
DECLARE_mInt32(in_memory_file_size);
1003
1004
// ParquetReaderWrap prefetch buffer size
1005
DECLARE_Int32(parquet_reader_max_buffer_size);
1006
// Max size of parquet page header in bytes
1007
DECLARE_mInt32(parquet_header_max_size_mb);
1008
// Max buffer size for parquet row group
1009
DECLARE_mInt32(parquet_rowgroup_max_buffer_mb);
1010
// Max buffer size for parquet chunk column
1011
DECLARE_mInt32(parquet_column_max_buffer_mb);
1012
// Merge small IO, the max amplified read ratio
1013
DECLARE_mDouble(max_amplified_read_ratio);
1014
// Equivalent min size of each IO that can reach the maximum storage speed limit
1015
// 1MB for oss, 8KB for hdfs
1016
DECLARE_mInt32(merged_oss_min_io_size);
1017
DECLARE_mInt32(merged_hdfs_min_io_size);
1018
1019
// OrcReader
1020
DECLARE_mInt32(orc_natural_read_size_mb);
1021
DECLARE_mInt64(big_column_size_buffer);
1022
DECLARE_mInt64(small_column_size_buffer);
1023
1024
DECLARE_mInt32(runtime_filter_sampling_frequency);
1025
DECLARE_mInt32(execution_max_rpc_timeout_sec);
1026
DECLARE_mBool(execution_ignore_eovercrowded);
1027
1028
// cooldown task configs
1029
DECLARE_Int32(cooldown_thread_num);
1030
DECLARE_mInt64(generate_cooldown_task_interval_sec);
1031
DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h
1032
DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
1033
DECLARE_Int32(cold_data_compaction_thread_num);
1034
DECLARE_mInt32(cold_data_compaction_interval_sec);
1035
1036
DECLARE_Int32(min_s3_file_system_thread_num);
1037
DECLARE_Int32(max_s3_file_system_thread_num);
1038
1039
DECLARE_Bool(enable_time_lut);
1040
DECLARE_mBool(enable_simdjson_reader);
1041
1042
DECLARE_mBool(enable_query_like_bloom_filter);
1043
// number of s3 scanner thread pool size
1044
DECLARE_Int32(doris_remote_scanner_thread_pool_thread_num);
1045
// number of s3 scanner thread pool queue size
1046
DECLARE_Int32(doris_remote_scanner_thread_pool_queue_size);
1047
1048
// limit the queue of pending batches which will be sent by a single nodechannel
1049
DECLARE_mInt64(nodechannel_pending_queue_max_bytes);
1050
1051
// The batch size for sending data by brpc streaming client
1052
DECLARE_mInt64(brpc_streaming_client_batch_bytes);
1053
DECLARE_mInt64(block_cache_wait_timeout_ms);
1054
1055
DECLARE_Bool(enable_brpc_builtin_services);
1056
1057
// Max waiting time to wait the "plan fragment start" rpc.
1058
// If timeout, the fragment will be cancelled.
1059
// This parameter is usually only used when the FE loses connection,
1060
// and the BE can automatically cancel the relevant fragment after the timeout,
1061
// so as to avoid occupying the execution thread for a long time.
1062
DECLARE_mInt32(max_fragment_start_wait_time_seconds);
1063
1064
// Node role tag for backend. Mix role is the default role, and computation role have no
1065
// any tablet.
1066
DECLARE_String(be_node_role);
1067
1068
// Hide webserver page for safety.
1069
// Hide the be config page for webserver.
1070
DECLARE_Bool(hide_webserver_config_page);
1071
1072
DECLARE_Bool(enable_segcompaction);
1073
1074
// Max number of segments allowed in a single segcompaction task.
1075
DECLARE_Int32(segcompaction_batch_size);
1076
1077
// Max row count allowed in a single source segment, bigger segments will be skipped.
1078
DECLARE_Int32(segcompaction_candidate_max_rows);
1079
1080
// Max file size allowed in a single source segment, bigger segments will be skipped.
1081
DECLARE_Int64(segcompaction_candidate_max_bytes);
1082
1083
// Max total row count allowed in a single segcompaction task.
1084
DECLARE_Int32(segcompaction_task_max_rows);
1085
1086
// Max total file size allowed in a single segcompaction task.
1087
DECLARE_Int64(segcompaction_task_max_bytes);
1088
1089
// Global segcompaction thread pool size.
1090
DECLARE_mInt32(segcompaction_num_threads);
1091
1092
// enable java udf and jdbc scannode
1093
DECLARE_Bool(enable_java_support);
1094
1095
// enable prefetch tablets before opening
1096
DECLARE_mBool(enable_prefetch_tablet);
1097
1098
// Set config randomly to check more issues in github workflow
1099
DECLARE_Bool(enable_fuzzy_mode);
1100
1101
DECLARE_Bool(enable_debug_points);
1102
1103
DECLARE_Int32(pipeline_executor_size);
1104
1105
// block file cache
1106
DECLARE_Bool(enable_file_cache);
1107
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
1108
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
1109
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}]
1110
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
1111
// Note1: storage is "disk" by default
1112
// Note2: when the storage is "memory", the path is ignored. So you can set xxx to anything you like
1113
// and doris will just reset the path to "memory" internally.
1114
// In a very wierd case when your storage is disk, and the directory, by accident, is named
1115
// "memory" for some reason, you should write the path as:
1116
//     {"path": "memory", "total_size":53687091200, "storage": "disk"}
1117
// or use the default storage value:
1118
//     {"path": "memory", "total_size":53687091200}
1119
// Both will use the directory "memory" on the disk instead of the real RAM.
1120
DECLARE_String(file_cache_path);
1121
DECLARE_Int64(file_cache_each_block_size);
1122
DECLARE_Bool(clear_file_cache);
1123
DECLARE_Bool(enable_file_cache_query_limit);
1124
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
1125
DECLARE_Int32(file_cache_exit_disk_resource_limit_mode_percent);
1126
DECLARE_mBool(enable_evict_file_cache_in_advance);
1127
DECLARE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent);
1128
DECLARE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent);
1129
DECLARE_mInt32(file_cache_evict_in_advance_interval_ms);
1130
DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes);
1131
DECLARE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold);
1132
DECLARE_mBool(enable_read_cache_file_directly);
1133
DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size);
1134
// If true, evict the ttl cache using LRU when full.
1135
// Otherwise, only expiration can evict ttl and new data won't add to cache when full.
1136
DECLARE_Bool(enable_ttl_cache_evict_using_lru);
1137
DECLARE_mBool(enbale_dump_error_file);
1138
// limit the max size of error log on disk
1139
DECLARE_mInt64(file_cache_error_log_limit_bytes);
1140
DECLARE_mInt64(cache_lock_wait_long_tail_threshold_us);
1141
DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
1142
// Base compaction may retrieve and produce some less frequently accessed data,
1143
// potentially affecting the file cache hit rate.
1144
// This configuration determines whether to retain the output within the file cache.
1145
// Make your choice based on the following considerations:
1146
// If your file cache is ample enough to accommodate all the data in your database,
1147
// enable this option; otherwise, it is recommended to leave it disabled.
1148
DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
1149
DECLARE_mInt64(file_cache_remove_block_qps_limit);
1150
DECLARE_mInt64(file_cache_background_gc_interval_ms);
1151
DECLARE_mInt64(file_cache_background_monitor_interval_ms);
1152
DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms);
1153
DECLARE_mInt64(file_cache_background_ttl_gc_batch);
1154
1155
// inverted index searcher cache
1156
// cache entry stay time after lookup
1157
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
1158
// cache entry that have not been visited for a certain period of time can be cleaned up by GC thread
1159
DECLARE_mInt32(inverted_index_cache_stale_sweep_time_sec);
1160
// inverted index searcher cache size
1161
DECLARE_String(inverted_index_searcher_cache_limit);
1162
// set `true` to enable insert searcher into cache when write inverted index data
1163
DECLARE_Bool(enable_write_index_searcher_cache);
1164
DECLARE_Bool(enable_inverted_index_cache_check_timestamp);
1165
DECLARE_mBool(enable_inverted_index_correct_term_write);
1166
DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50%
1167
DECLARE_Int32(inverted_index_query_cache_shards);
1168
1169
// inverted index match bitmap cache size
1170
DECLARE_String(inverted_index_query_cache_limit);
1171
1172
// inverted index
1173
DECLARE_mDouble(inverted_index_ram_buffer_size);
1174
DECLARE_mInt32(inverted_index_max_buffered_docs);
1175
// dict path for chinese analyzer
1176
DECLARE_String(inverted_index_dict_path);
1177
DECLARE_Int32(inverted_index_read_buffer_size);
1178
// tree depth for bkd index
1179
DECLARE_Int32(max_depth_in_bkd_tree);
1180
// index compaction
1181
DECLARE_mBool(inverted_index_compaction_enable);
1182
// Only for debug, do not use in production
1183
DECLARE_mBool(debug_inverted_index_compaction);
1184
// index by RAM directory
1185
DECLARE_mBool(inverted_index_ram_dir_enable);
1186
// use num_broadcast_buffer blocks as buffer to do broadcast
1187
DECLARE_Int32(num_broadcast_buffer);
1188
1189
// max depth of expression tree allowed.
1190
DECLARE_Int32(max_depth_of_expr_tree);
1191
1192
// Report a tablet as bad when io errors occurs more than this value.
1193
DECLARE_mInt64(max_tablet_io_errors);
1194
1195
// Report a tablet as bad when its path not found
1196
DECLARE_Int32(tablet_path_check_interval_seconds);
1197
DECLARE_mInt32(tablet_path_check_batch_size);
1198
1199
// it must be larger than or equal to 5MB
1200
DECLARE_mInt64(s3_write_buffer_size);
1201
// Log interval when doing s3 upload task
1202
DECLARE_mInt32(s3_file_writer_log_interval_second);
1203
// the max number of cached file handle for block segemnt
1204
DECLARE_mInt64(file_cache_max_file_reader_cache_size);
1205
DECLARE_mInt64(hdfs_write_batch_buffer_size_mb);
1206
//enable shrink memory
1207
DECLARE_mBool(enable_shrink_memory);
1208
// enable cache for high concurrent point query work load
1209
DECLARE_mInt32(schema_cache_capacity);
1210
DECLARE_mInt32(schema_cache_sweep_time_sec);
1211
1212
// max number of segment cache
1213
DECLARE_Int32(segment_cache_capacity);
1214
DECLARE_Int32(segment_cache_fd_percentage);
1215
DECLARE_Int32(segment_cache_memory_percentage);
1216
DECLARE_mInt32(estimated_mem_per_column_reader);
1217
1218
// enable binlog
1219
DECLARE_Bool(enable_feature_binlog);
1220
1221
// enable set in BitmapValue
1222
DECLARE_Bool(enable_set_in_bitmap_value);
1223
1224
// max number of hdfs file handle in cache
1225
DECLARE_Int64(max_hdfs_file_handle_cache_num);
1226
DECLARE_Int32(max_hdfs_file_handle_cache_time_sec);
1227
1228
// max number of meta info of external files, such as parquet footer
1229
DECLARE_Int64(max_external_file_meta_cache_num);
1230
// Apply delete pred in cumu compaction
1231
DECLARE_mBool(enable_delete_when_cumu_compaction);
1232
1233
// max_write_buffer_number for rocksdb
1234
DECLARE_Int32(rocksdb_max_write_buffer_number);
1235
1236
// Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false.
1237
DECLARE_mBool(allow_zero_date);
1238
// Allow invalid decimalv2 literal for compatible with old version. Recommend set it false strongly.
1239
DECLARE_mBool(allow_invalid_decimalv2_literal);
1240
// Allow to specify kerberos credentials cache path.
1241
DECLARE_mString(kerberos_ccache_path);
1242
// set krb5.conf path, use "/etc/krb5.conf" by default
1243
DECLARE_mString(kerberos_krb5_conf_path);
1244
1245
// Values include `none`, `glog`, `boost`, `glibc`, `libunwind`
1246
DECLARE_mString(get_stack_trace_tool);
1247
DECLARE_mBool(enable_address_sanitizers_with_stack_trace);
1248
1249
// DISABLED: Don't resolve location info.
1250
// FAST: Perform CU lookup using .debug_aranges (might be incomplete).
1251
// FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure.
1252
// FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL.
1253
DECLARE_mString(dwarf_location_info_mode);
1254
1255
// the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer
1256
DECLARE_mInt64(auto_inc_prefetch_size_ratio);
1257
1258
// the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer
1259
DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
1260
1261
// number of threads that fetch auto-inc ranges from FE
1262
DECLARE_mInt64(auto_inc_fetch_thread_num);
1263
// Max connection cache num for point lookup queries
1264
DECLARE_mInt64(lookup_connection_cache_capacity);
1265
1266
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
1267
DECLARE_mInt64(LZ4_HC_compression_level);
1268
// Threshold of a column as sparse column
1269
// Notice: TEST ONLY
1270
DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column);
1271
DECLARE_mBool(variant_use_cloud_schema_dict_cache);
1272
// Threshold to estimate a column is sparsed
1273
// Notice: TEST ONLY
1274
DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column);
1275
// Treat invalid json format str as string, instead of throwing exception if false
1276
DECLARE_mBool(variant_throw_exeception_on_invalid_json);
1277
1278
DECLARE_mBool(enable_merge_on_write_correctness_check);
1279
// USED FOR DEBUGING
1280
// core directly if the compaction found there's duplicate key on mow table
1281
DECLARE_mBool(enable_mow_compaction_correctness_check_core);
1282
// rowid conversion correctness check when compaction for mow table
1283
DECLARE_mBool(enable_rowid_conversion_correctness_check);
1284
// missing rows correctness check when compaction for mow table
1285
DECLARE_mBool(enable_missing_rows_correctness_check);
1286
// When the number of missing versions is more than this value, do not directly
1287
// retry the publish and handle it through async publish.
1288
DECLARE_mInt32(mow_publish_max_discontinuous_version_num);
1289
// When the version is not continuous for MOW table in publish phase and the gap between
1290
// current txn's publishing version and the max version of the tablet exceeds this value,
1291
// don't print warning log
1292
DECLARE_mInt32(publish_version_gap_logging_threshold);
1293
// get agg by cache for mow table
1294
DECLARE_mBool(enable_mow_get_agg_by_cache);
1295
// get agg correctness check for mow table
1296
DECLARE_mBool(enable_mow_get_agg_correctness_check_core);
1297
1298
// The secure path with user files, used in the `local` table function.
1299
DECLARE_mString(user_files_secure_path);
1300
1301
// If fe's frontend info has not been updated for more than fe_expire_duration_seconds, it will be regarded
1302
// as an abnormal fe, this will cause be to cancel this fe's related query.
1303
DECLARE_Int32(fe_expire_duration_seconds);
1304
1305
// If use stop_be.sh --grace, then BE has to wait all running queries to stop to avoiding running query failure
1306
// , but if the waiting time exceed the limit, then be will exit directly.
1307
// During this period, FE will not send any queries to BE and waiting for all running queries to stop.
1308
DECLARE_Int32(grace_shutdown_wait_seconds);
1309
1310
// BitmapValue serialize version.
1311
DECLARE_Int16(bitmap_serialize_version);
1312
1313
// group commit config
1314
DECLARE_String(group_commit_wal_path);
1315
DECLARE_Int32(group_commit_replay_wal_retry_num);
1316
DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds);
1317
DECLARE_Int32(group_commit_replay_wal_retry_interval_max_seconds);
1318
DECLARE_mInt32(group_commit_relay_wal_threads);
1319
// This config can be set to limit thread number in group commit request fragment thread pool.
1320
DECLARE_mInt32(group_commit_insert_threads);
1321
DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
1322
DECLARE_Bool(wait_internal_group_commit_finish);
1323
// Max size(bytes) of group commit queues, used for mem back pressure.
1324
DECLARE_mInt32(group_commit_queue_mem_limit);
1325
// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
1326
// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
1327
DECLARE_mString(group_commit_wal_max_disk_limit);
1328
DECLARE_Bool(group_commit_wait_replay_wal_finish);
1329
1330
// The configuration item is used to lower the priority of the scanner thread,
1331
// typically employed to ensure CPU scheduling for write operations.
1332
// Default is 0, which is default value of thread nice value, increase this value
1333
// to lower the priority of scan threads
1334
DECLARE_Int32(scan_thread_nice_value);
1335
// Used to modify the recycle interval of tablet schema cache
1336
DECLARE_mInt32(tablet_schema_cache_recycle_interval);
1337
// Granularity is at the column level
1338
DECLARE_mInt32(tablet_schema_cache_capacity);
1339
1340
// Use `LOG(FATAL)` to replace `throw` when true
1341
DECLARE_mBool(exit_on_exception);
1342
1343
// cgroup
1344
DECLARE_String(doris_cgroup_cpu_path);
1345
DECLARE_mBool(enable_be_proc_monitor);
1346
DECLARE_mInt32(be_proc_monitor_interval_ms);
1347
DECLARE_Int32(workload_group_metrics_interval_ms);
1348
1349
DECLARE_mBool(enable_workload_group_memory_gc);
1350
1351
// This config controls whether the s3 file writer would flush cache asynchronously
1352
DECLARE_Bool(enable_flush_file_cache_async);
1353
1354
// Remove predicate that is always true for a segment.
1355
DECLARE_Bool(ignore_always_true_predicate_for_segment);
1356
1357
// Ingest binlog work pool size
1358
DECLARE_Int32(ingest_binlog_work_pool_size);
1359
1360
// Ingest binlog with persistent connection
1361
DECLARE_Bool(enable_ingest_binlog_with_persistent_connection);
1362
1363
// Download binlog rate limit, unit is KB/s
1364
DECLARE_Int32(download_binlog_rate_limit_kbs);
1365
1366
DECLARE_mInt32(buffered_reader_read_timeout_ms);
1367
1368
// whether to enable /api/snapshot api
1369
DECLARE_Bool(enable_snapshot_action);
1370
1371
// The max columns size for a tablet schema
1372
DECLARE_mInt32(variant_max_merged_tablet_schema_size);
1373
1374
DECLARE_mInt64(local_exchange_buffer_mem_limit);
1375
1376
DECLARE_mInt64(enable_debug_log_timeout_secs);
1377
1378
DECLARE_mBool(enable_column_type_check);
1379
1380
// Tolerance for the number of partition id 0 in rowset, default 0
1381
DECLARE_Int32(ignore_invalid_partition_id_rowset_num);
1382
1383
DECLARE_mInt32(report_query_statistics_interval_ms);
1384
DECLARE_mInt32(query_statistics_reserve_timeout_ms);
1385
DECLARE_mInt32(report_exec_status_thread_num);
1386
1387
// consider two high usage disk at the same available level if they do not exceed this diff.
1388
DECLARE_mDouble(high_disk_avail_level_diff_usages);
1389
1390
// create tablet in partition random robin idx lru size, default 10000
1391
DECLARE_Int32(partition_disk_index_lru_size);
1392
DECLARE_String(spill_storage_root_path);
1393
// Spill storage limit specified as number of bytes
1394
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
1395
// or percentage of capaity ('<int>%').
1396
// Defaults to bytes if no unit is given.
1397
// Must larger than 0.
1398
// If specified as percentage, the final limit value is:
1399
//   disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit
1400
DECLARE_String(spill_storage_limit);
1401
DECLARE_mInt32(spill_gc_interval_ms);
1402
DECLARE_mInt32(spill_gc_work_time_ms);
1403
DECLARE_Int32(spill_io_thread_pool_thread_num);
1404
DECLARE_Int32(spill_io_thread_pool_queue_size);
1405
1406
DECLARE_mBool(check_segment_when_build_rowset_meta);
1407
1408
DECLARE_Int32(num_query_ctx_map_partitions);
1409
1410
DECLARE_mBool(force_azure_blob_global_endpoint);
1411
1412
DECLARE_mBool(enable_s3_rate_limiter);
1413
DECLARE_mInt64(s3_get_bucket_tokens);
1414
DECLARE_mInt64(s3_get_token_per_second);
1415
DECLARE_mInt64(s3_get_token_limit);
1416
1417
DECLARE_mInt64(s3_put_bucket_tokens);
1418
DECLARE_mInt64(s3_put_token_per_second);
1419
DECLARE_mInt64(s3_put_token_limit);
1420
// max s3 client retry times
1421
DECLARE_mInt32(max_s3_client_retry);
1422
// When meet s3 429 error, the "get" request will
1423
// sleep s3_read_base_wait_time_ms (*1, *2, *3, *4) ms
1424
// get try again.
1425
// The max sleep time is s3_read_max_wait_time_ms
1426
// and the max retry time is max_s3_client_retry
1427
DECLARE_mInt32(s3_read_base_wait_time_ms);
1428
DECLARE_mInt32(s3_read_max_wait_time_ms);
1429
1430
// write as inverted index tmp directory
1431
DECLARE_String(tmp_file_dir);
1432
1433
// the directory for storing the trino-connector plugins.
1434
DECLARE_String(trino_connector_plugin_dir);
1435
1436
// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client
1437
DECLARE_mString(ca_cert_file_paths);
1438
1439
/** Table sink configurations(currently contains only external table types) **/
1440
// Minimum data processed to scale writers in exchange when non partition writing
1441
DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold);
1442
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1443
DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold);
1444
// Minimum partition data processed to rebalance writers in exchange when partition writing
1445
DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold);
1446
// Maximum processed partition nums of per writer when partition writing
1447
DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer);
1448
1449
/** Hive sink configurations **/
1450
DECLARE_mInt64(hive_sink_max_file_size);
1451
1452
/** Iceberg sink configurations **/
1453
DECLARE_mInt64(iceberg_sink_max_file_size);
1454
1455
// Number of open tries, default 1 means only try to open once.
1456
// Retry the Open num_retries time waiting 100 milliseconds between retries.
1457
DECLARE_mInt32(thrift_client_open_num_tries);
1458
1459
// http scheme in S3Client to use. E.g. http or https
1460
DECLARE_String(s3_client_http_scheme);
1461
1462
DECLARE_mBool(ignore_schema_change_check);
1463
1464
/** Only use in fuzzy test **/
1465
DECLARE_mInt64(string_overflow_size);
1466
1467
// The min thread num for BufferedReaderPrefetchThreadPool
1468
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread);
1469
// The max thread num for BufferedReaderPrefetchThreadPool
1470
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread);
1471
// The min thread num for S3FileUploadThreadPool
1472
DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread);
1473
// The max thread num for S3FileUploadThreadPool
1474
DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread);
1475
// The maximum jvm heap usage ratio for hdfs write workload
1476
DECLARE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio);
1477
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
1478
DECLARE_mInt64(hdfs_jni_write_sleep_milliseconds);
1479
// The max retry times when hdfs write failed
1480
DECLARE_mInt64(hdfs_jni_write_max_retry_time);
1481
1482
// The min thread num for NonBlockCloseThreadPool
1483
DECLARE_Int64(min_nonblock_close_thread_num);
1484
// The max thread num for NonBlockCloseThreadPool
1485
DECLARE_Int64(max_nonblock_close_thread_num);
1486
// The possibility that mem allocator throws an exception during memory allocation
1487
// This config is for test usage, be careful when changing it.
1488
DECLARE_mDouble(mem_alloc_fault_probability);
1489
// The time out milliseconds for remote fetch schema RPC
1490
DECLARE_mInt64(fetch_remote_schema_rpc_timeout_ms);
1491
// The size of the local buffer for S3FileSytem's upload function
1492
1493
DECLARE_Int64(s3_file_system_local_upload_buffer_size);
1494
1495
//JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues.
1496
DECLARE_Bool(enable_jvm_monitor);
1497
1498
// Num threads to load data dirs, default value -1 indicates the same number of threads as the number of data dirs
1499
DECLARE_Int32(load_data_dirs_threads);
1500
1501
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
1502
DECLARE_mBool(skip_loading_stale_rowset_meta);
1503
// Whether to use file to record log. When starting BE with --console,
1504
// all logs will be written to both standard output and file.
1505
// Disable this option will no longer use file to record log.
1506
// Only works when starting BE with --console.
1507
DECLARE_Bool(enable_file_logger);
1508
1509
// The minimum row group size when exporting Parquet files.
1510
DECLARE_Int64(min_row_group_size);
1511
1512
DECLARE_mInt64(compaction_memory_bytes_limit);
1513
1514
DECLARE_mInt64(compaction_batch_size);
1515
1516
DECLARE_mBool(enable_parquet_page_index);
1517
1518
// Wheather to ignore not found file in external teble(eg, hive)
1519
// Default is true, if set to false, the not found file will result in query failure.
1520
DECLARE_mBool(ignore_not_found_file_in_external_table);
1521
1522
DECLARE_mBool(enable_hdfs_mem_limiter);
1523
1524
// Define how many percent data in hashtable bigger than limit
1525
// we should do agg limit opt
1526
DECLARE_mInt16(topn_agg_limit_multiplier);
1527
1528
DECLARE_mInt64(tablet_meta_serialize_size_limit);
1529
1530
DECLARE_mInt64(pipeline_task_leakage_detect_period_secs);
1531
// To be compatible with hadoop's block compression
1532
DECLARE_mInt32(snappy_compression_block_size);
1533
DECLARE_mInt32(lz4_compression_block_size);
1534
1535
DECLARE_mBool(enable_pipeline_task_leakage_detect);
1536
1537
// MB
1538
DECLARE_Int32(query_cache_size);
1539
DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
1540
1541
DECLARE_mBool(enable_delete_bitmap_merge_on_compaction);
1542
1543
// Enable validation to check the correctness of table size.
1544
DECLARE_Bool(enable_table_size_correctness_check);
1545
// Enable sleep 5s between delete cumulative compaction.
1546
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
1547
1548
DECLARE_mInt32(compaction_num_per_round);
1549
1550
DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds);
1551
DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
1552
DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
1553
DECLARE_mInt32(schema_dict_cache_capacity);
1554
1555
DECLARE_mBool(enable_mow_verbose_log);
1556
1557
// whether to prune rows with delete sign = 1 in base compaction
1558
// ATTN: this config is only for test
1559
DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
1560
1561
// Because the root_path for the HDFS resource was previously passed an empty string (since v2.1),
1562
// which was incorrect, this configuration has been added to ensure compatibility
1563
// and guarantee that the root_path works as expected.
1564
DECLARE_Bool(enable_root_path_of_hdfs_resource);
1565
1566
DECLARE_mInt32(tablet_sched_delay_time_ms);
1567
DECLARE_mInt32(load_trigger_compaction_version_percent);
1568
DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
1569
DECLARE_mBool(enable_compaction_pause_on_high_memory);
1570
1571
#ifdef BE_TEST
1572
// test s3
1573
DECLARE_String(test_s3_resource);
1574
DECLARE_String(test_s3_ak);
1575
DECLARE_String(test_s3_sk);
1576
DECLARE_String(test_s3_endpoint);
1577
DECLARE_String(test_s3_region);
1578
DECLARE_String(test_s3_bucket);
1579
DECLARE_String(test_s3_prefix);
1580
#endif
1581
1582
class Register {
1583
public:
1584
    struct Field {
1585
        const char* type = nullptr;
1586
        const char* name = nullptr;
1587
        void* storage = nullptr;
1588
        const char* defval = nullptr;
1589
        bool valmutable = false;
1590
        Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1591
              bool fvalmutable)
1592
                : type(ftype),
1593
                  name(fname),
1594
                  storage(fstorage),
1595
                  defval(fdefval),
1596
669
                  valmutable(fvalmutable) {}
1597
    };
1598
1599
public:
1600
    static std::map<std::string, Field>* _s_field_map;
1601
1602
public:
1603
    Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval,
1604
665
             bool fvalmutable) {
1605
665
        if (_s_field_map == nullptr) {
1606
1
            _s_field_map = new std::map<std::string, Field>();
1607
1
        }
1608
665
        Field field(ftype, fname, fstorage, fdefval, fvalmutable);
1609
665
        _s_field_map->insert(std::make_pair(std::string(fname), field));
1610
665
    }
1611
};
1612
1613
// RegisterConfValidator class is used to store validator function of registered config fields in
1614
// Register::_s_field_map.
1615
// If any validator return false when BE bootstart, the bootstart will be terminated.
1616
// If validator return false when use http API to update some config, the config will not
1617
// be modified and the API will return failure.
1618
class RegisterConfValidator {
1619
public:
1620
    // Validator for each config name.
1621
    static std::map<std::string, std::function<bool()>>* _s_field_validator;
1622
1623
public:
1624
    RegisterConfValidator(const char* fname, const std::function<bool()>& validator) {
1625
        if (_s_field_validator == nullptr) {
1626
            _s_field_validator = new std::map<std::string, std::function<bool()>>();
1627
        }
1628
        // register validator to _s_field_validator
1629
        _s_field_validator->insert(std::make_pair(std::string(fname), validator));
1630
    }
1631
};
1632
1633
// configuration properties load from config file.
1634
class Properties {
1635
public:
1636
    // load conf from file, if must_exist is true and file does not exist, return false
1637
    bool load(const char* conf_file, bool must_exist = true);
1638
1639
    // Find the config value by key from `file_conf_map`.
1640
    // If found, set `retval` to the config value,
1641
    // or set `retval` to `defstr`
1642
    // if retval is not set(in case defstr is nullptr), set is_retval_set to false
1643
    template <typename T>
1644
    bool get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set,
1645
                        std::string& rawval) const;
1646
1647
    void set(const std::string& key, const std::string& val);
1648
1649
    void set_force(const std::string& key, const std::string& val);
1650
1651
    // dump props to conf file
1652
    Status dump(const std::string& conffile);
1653
1654
private:
1655
    std::map<std::string, std::string> file_conf_map;
1656
};
1657
1658
// full configurations.
1659
extern std::map<std::string, std::string>* full_conf_map;
1660
1661
extern std::mutex custom_conf_lock;
1662
1663
// Init the config from `conf_file`.
1664
// If fill_conf_map is true, the updated config will also update the `full_conf_map`.
1665
// If must_exist is true and `conf_file` does not exist, this function will return false.
1666
// If set_to_default is true, the config value will be set to default value if not found in `conf_file`.
1667
bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist = true,
1668
          bool set_to_default = true);
1669
1670
Status set_config(const std::string& field, const std::string& value, bool need_persist = false,
1671
                  bool force = false);
1672
1673
Status persist_config(const std::string& field, const std::string& value);
1674
1675
std::mutex* get_mutable_string_config_lock();
1676
1677
std::vector<std::vector<std::string>> get_config_info();
1678
1679
Status set_fuzzy_configs();
1680
1681
void update_config(const std::string& field, const std::string& value);
1682
1683
} // namespace config
1684
} // namespace doris