/root/doris/be/src/common/config.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #pragma once |
19 | | |
20 | | #include <cstdint> |
21 | | #include <functional> |
22 | | #include <map> |
23 | | #include <mutex> |
24 | | #include <string> |
25 | | #include <utility> |
26 | | #include <vector> |
27 | | |
28 | | #define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME |
29 | | |
30 | | #define DECLARE_Bool(name) DECLARE_FIELD(bool, name) |
31 | | #define DECLARE_Int16(name) DECLARE_FIELD(int16_t, name) |
32 | | #define DECLARE_Int32(name) DECLARE_FIELD(int32_t, name) |
33 | | #define DECLARE_Int64(name) DECLARE_FIELD(int64_t, name) |
34 | | #define DECLARE_Double(name) DECLARE_FIELD(double, name) |
35 | | #define DECLARE_String(name) DECLARE_FIELD(std::string, name) |
36 | | #define DECLARE_Bools(name) DECLARE_FIELD(std::vector<bool>, name) |
37 | | #define DECLARE_Int16s(name) DECLARE_FIELD(std::vector<int16_t>, name) |
38 | | #define DECLARE_Int32s(name) DECLARE_FIELD(std::vector<int32_t>, name) |
39 | | #define DECLARE_Int64s(name) DECLARE_FIELD(std::vector<int64_t>, name) |
40 | | #define DECLARE_Doubles(name) DECLARE_FIELD(std::vector<double>, name) |
41 | | #define DECLARE_Strings(name) DECLARE_FIELD(std::vector<std::string>, name) |
42 | | #define DECLARE_mBool(name) DECLARE_FIELD(bool, name) |
43 | | #define DECLARE_mInt16(name) DECLARE_FIELD(int16_t, name) |
44 | | #define DECLARE_mInt32(name) DECLARE_FIELD(int32_t, name) |
45 | | #define DECLARE_mInt64(name) DECLARE_FIELD(int64_t, name) |
46 | | #define DECLARE_mDouble(name) DECLARE_FIELD(double, name) |
47 | | #define DECLARE_mString(name) DECLARE_FIELD(std::string, name) |
48 | | |
49 | | #define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE) \ |
50 | 21 | FIELD_TYPE FIELD_NAME; \ |
51 | 21 | static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &(FIELD_NAME), FIELD_DEFAULT, \ |
52 | 21 | VALMUTABLE); |
53 | | |
54 | | #define DEFINE_VALIDATOR(FIELD_NAME, VALIDATOR) \ |
55 | | static auto validator_##FIELD_NAME = VALIDATOR; \ |
56 | | static RegisterConfValidator reg_validator_##FIELD_NAME( \ |
57 | | #FIELD_NAME, []() -> bool { return validator_##FIELD_NAME(FIELD_NAME); }); |
58 | | |
59 | 1 | #define DEFINE_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false) |
60 | 1 | #define DEFINE_Bools(name, defaultstr) DEFINE_FIELD(std::vector<bool>, name, defaultstr, false) |
61 | 1 | #define DEFINE_Doubles(name, defaultstr) DEFINE_FIELD(std::vector<double>, name, defaultstr, false) |
62 | 1 | #define DEFINE_Int16s(name, defaultstr) DEFINE_FIELD(std::vector<int16_t>, name, defaultstr, false) |
63 | 1 | #define DEFINE_Int32s(name, defaultstr) DEFINE_FIELD(std::vector<int32_t>, name, defaultstr, false) |
64 | 1 | #define DEFINE_Int64s(name, defaultstr) DEFINE_FIELD(std::vector<int64_t>, name, defaultstr, false) |
65 | 3 | #define DEFINE_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false) |
66 | 1 | #define DEFINE_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false) |
67 | 1 | #define DEFINE_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false) |
68 | 1 | #define DEFINE_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false) |
69 | 2 | #define DEFINE_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false) |
70 | | #define DEFINE_Strings(name, defaultstr) \ |
71 | 1 | DEFINE_FIELD(std::vector<std::string>, name, defaultstr, false) |
72 | 1 | #define DEFINE_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true) |
73 | 1 | #define DEFINE_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true) |
74 | 1 | #define DEFINE_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true) |
75 | 1 | #define DEFINE_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true) |
76 | 1 | #define DEFINE_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true) |
77 | 1 | #define DEFINE_mString(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, true) |
78 | | #define DEFINE_Validator(name, validator) DEFINE_VALIDATOR(name, validator) |
79 | | |
80 | | namespace doris { |
81 | | class Status; |
82 | | |
83 | | // If you want to modify the value of config, please go to common/config.cpp to modify. |
84 | | namespace config { |
85 | | // Dir of custom config file |
86 | | DECLARE_String(custom_config_dir); |
87 | | |
88 | | // Dir of jdbc drivers |
89 | | DECLARE_String(jdbc_drivers_dir); |
90 | | |
91 | | // cluster id |
92 | | DECLARE_Int32(cluster_id); |
93 | | // port on which BackendService is exported |
94 | | DECLARE_Int32(be_port); |
95 | | |
96 | | // port for brpc |
97 | | DECLARE_Int32(brpc_port); |
98 | | |
99 | | // port for arrow flight sql |
100 | | // Default -1, do not start arrow flight sql server. |
101 | | DECLARE_Int32(arrow_flight_sql_port); |
102 | | |
103 | | // If the external client cannot directly access priority_networks, set public_host to be accessible |
104 | | // to external client. |
105 | | // There are usually two usage scenarios: |
106 | | // 1. in production environment, it is often inconvenient to expose Doris BE nodes to the external network. |
107 | | // However, a reverse proxy (such as Nginx) can be added to all Doris BE nodes, and the external client will be |
108 | | // randomly routed to a Doris BE node when connecting to Nginx. set public_host to the host of Nginx. |
109 | | // 2. if priority_networks is an internal network IP, and BE node has its own independent external IP, |
110 | | // but Doris currently does not support modifying priority_networks, setting public_host to the real external IP. |
111 | | DECLARE_mString(public_host); |
112 | | |
113 | | // If the BE node is connected to the external network through a reverse proxy like Nginx |
114 | | // and need to use Arrow Flight SQL, should add a server in Nginx to reverse proxy |
115 | | // `Nginx:arrow_flight_sql_proxy_port` to `BE_priority_networks:arrow_flight_sql_port`. For example: |
116 | | // upstream arrowflight { |
117 | | // server 10.16.10.8:8069; |
118 | | // server 10.16.10.8:8068; |
119 | | //} |
120 | | // server { |
121 | | // listen 8167 http2; |
122 | | // listen [::]:8167 http2; |
123 | | // server_name doris.arrowflight.com; |
124 | | // } |
125 | | DECLARE_Int32(arrow_flight_sql_proxy_port); |
126 | | |
127 | | // the number of bthreads for brpc, the default value is set to -1, |
128 | | // which means the number of bthreads is #cpu-cores |
129 | | DECLARE_Int32(brpc_num_threads); |
130 | | DECLARE_Int32(brpc_idle_timeout_sec); |
131 | | |
132 | | // Declare a selection strategy for those servers have many ips. |
133 | | // Note that there should at most one ip match this list. |
134 | | // This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24 |
135 | | // If no ip match this rule, will choose one randomly. |
136 | | DECLARE_String(priority_networks); |
137 | | |
138 | | // performance moderate or compact, only tcmalloc compile |
139 | | DECLARE_String(memory_mode); |
140 | | |
141 | | // if true, process memory limit and memory usage based on cgroup memory info. |
142 | | DECLARE_mBool(enable_use_cgroup_memory_info); |
143 | | |
144 | | // process memory limit specified as number of bytes |
145 | | // ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'), |
146 | | // or percentage of the physical memory ('<int>%'). |
147 | | // defaults to bytes if no unit is given" |
148 | | // must larger than 0. and if larger than physical memory size, |
149 | | // it will be set to physical memory size. |
150 | | DECLARE_String(mem_limit); |
151 | | |
152 | | // Soft memory limit as a fraction of hard memory limit. |
153 | | DECLARE_Double(soft_mem_limit_frac); |
154 | | |
155 | | // Schema change memory limit as a fraction of soft memory limit. |
156 | | DECLARE_Double(schema_change_mem_limit_frac); |
157 | | |
158 | | // Many modern allocators (for example) do not do a mremap for |
159 | | // realloc, even in case of large enough chunks of memory. Although this allows |
160 | | // you to increase performance and reduce memory consumption during realloc. |
161 | | // To fix this, we do mremap manually if the chunk of memory is large enough. |
162 | | // |
163 | | // The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address |
164 | | // space is very slow, especially in the case of a large number of threads. We |
165 | | // expect that the set of operations mmap/something to do/mremap can only be |
166 | | // performed about 1000 times per second. |
167 | | // |
168 | | // P.S. This is also required, because tcmalloc can not allocate a chunk of |
169 | | // memory greater than 16 GB. |
170 | | DECLARE_mInt64(mmap_threshold); // bytes |
171 | | |
172 | | // When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied. |
173 | | // Increase can reduce the number of hash table resize, but may waste more memory. |
174 | | DECLARE_mInt32(hash_table_double_grow_degree); |
175 | | |
176 | | // The max fill rate for hash table |
177 | | DECLARE_mInt32(max_fill_rate); |
178 | | |
179 | | DECLARE_mInt32(double_resize_threshold); |
180 | | |
181 | | // The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 6.4G, |
182 | | // actual low water mark=min(6.4G, MemTotal * 5%), avoid wasting too much memory on machines |
183 | | // with large memory larger than 128G. |
184 | | // Turn up max. On machines with more than 128G memory, more memory buffers will be reserved for Full GC. |
185 | | // Turn down max. will use as much memory as possible. |
186 | | DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes); |
187 | | |
188 | | // reserve a small amount of memory so we do not trigger MinorGC |
189 | | DECLARE_Int64(memtable_limiter_reserved_memory_bytes); |
190 | | |
191 | | // The size of the memory that gc wants to release each time, as a percentage of the mem limit. |
192 | | DECLARE_mString(process_minor_gc_size); |
193 | | DECLARE_mString(process_full_gc_size); |
194 | | |
195 | | // If true, when the process does not exceed the soft mem limit, the query memory will not be limited; |
196 | | // when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently |
197 | | // used memory and the exec_mem_limit will be canceled. |
198 | | // If false, cancel query when the memory used exceeds exec_mem_limit, same as before. |
199 | | DECLARE_mBool(enable_query_memory_overcommit); |
200 | | |
201 | | // gc will release cache, cancel task, and task will wait for gc to release memory, |
202 | | // default gc strategy is conservative, if you want to exclude the interference of gc, let it be true |
203 | | DECLARE_mBool(disable_memory_gc); |
204 | | |
205 | | // when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G, |
206 | | // if alloc successful, will print a warning with stacktrace, but not prevent memory alloc. |
207 | | // if alloc failed using Doris Allocator, will print stacktrace in error log. |
208 | | // if is -1, disable print stacktrace when alloc large memory. |
209 | | DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes); |
210 | | |
211 | | // when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled. |
212 | | // if you need a core dump to analyze large memory allocation, |
213 | | // modify this parameter to crash when large memory allocation occur will help |
214 | | DECLARE_mInt64(crash_in_alloc_large_memory_bytes); |
215 | | |
216 | | // If memory tracker value is inaccurate, BE will crash. usually used in test environments, default value is false. |
217 | | DECLARE_mBool(crash_in_memory_tracker_inaccurate); |
218 | | |
219 | | // default is true. if any memory tracking in Orphan mem tracker will report error. |
220 | | // !! not modify the default value of this conf!! otherwise memory errors cannot be detected in time. |
221 | | // allocator free memory not need to check, because when the thread memory tracker label is Orphan, |
222 | | // use the tracker saved in Allocator. |
223 | | DECLARE_mBool(enable_memory_orphan_check); |
224 | | |
225 | | // The maximum time a thread waits for a full GC. Currently only query will wait for full gc. |
226 | | DECLARE_mInt32(thread_wait_gc_max_milliseconds); |
227 | | |
228 | | // reach mem limit, don't serialize in batch |
229 | | DECLARE_mInt64(pre_serialize_keys_limit_bytes); |
230 | | |
231 | | // the port heartbeat service used |
232 | | DECLARE_Int32(heartbeat_service_port); |
233 | | // the count of heart beat service |
234 | | DECLARE_Int32(heartbeat_service_thread_count); |
235 | | // the count of thread to create table |
236 | | DECLARE_Int32(create_tablet_worker_count); |
237 | | // the count of thread to drop table |
238 | | DECLARE_Int32(drop_tablet_worker_count); |
239 | | // the count of thread to batch load |
240 | | DECLARE_Int32(push_worker_count_normal_priority); |
241 | | // the count of thread to high priority batch load |
242 | | DECLARE_Int32(push_worker_count_high_priority); |
243 | | // the count of thread to publish version |
244 | | DECLARE_Int32(publish_version_worker_count); |
245 | | // the count of tablet thread to publish version |
246 | | DECLARE_Int32(tablet_publish_txn_max_thread); |
247 | | // the timeout of EnginPublishVersionTask |
248 | | DECLARE_Int32(publish_version_task_timeout_s); |
249 | | // the count of thread to calc delete bitmap |
250 | | DECLARE_Int32(calc_delete_bitmap_max_thread); |
251 | | // the count of thread to clear transaction task |
252 | | DECLARE_Int32(clear_transaction_task_worker_count); |
253 | | // the count of thread to delete |
254 | | DECLARE_Int32(delete_worker_count); |
255 | | // the count of thread to alter table |
256 | | DECLARE_Int32(alter_tablet_worker_count); |
257 | | // the count of thread to alter index |
258 | | DECLARE_Int32(alter_index_worker_count); |
259 | | // the count of thread to clone |
260 | | DECLARE_Int32(clone_worker_count); |
261 | | // the count of thread to clone |
262 | | DECLARE_Int32(storage_medium_migrate_count); |
263 | | // the count of thread to check consistency |
264 | | DECLARE_Int32(check_consistency_worker_count); |
265 | | // the count of thread to upload |
266 | | DECLARE_Int32(upload_worker_count); |
267 | | // the count of thread to download |
268 | | DECLARE_Int32(download_worker_count); |
269 | | // the count of thread to make snapshot |
270 | | DECLARE_Int32(make_snapshot_worker_count); |
271 | | // the count of thread to release snapshot |
272 | | DECLARE_Int32(release_snapshot_worker_count); |
273 | | // report random wait a little time to avoid FE receiving multiple be reports at the same time. |
274 | | // do not set it to false for production environment |
275 | | DECLARE_mBool(report_random_wait); |
276 | | // the interval time(seconds) for agent report tasks signature to FE |
277 | | DECLARE_mInt32(report_task_interval_seconds); |
278 | | // the interval time(seconds) for refresh storage policy from FE |
279 | | DECLARE_mInt32(storage_refresh_storage_policy_task_interval_seconds); |
280 | | // the interval time(seconds) for agent report disk state to FE |
281 | | DECLARE_mInt32(report_disk_state_interval_seconds); |
282 | | // the interval time(seconds) for agent report olap table to FE |
283 | | DECLARE_mInt32(report_tablet_interval_seconds); |
284 | | // the max download speed(KB/s) |
285 | | DECLARE_mInt32(max_download_speed_kbps); |
286 | | // download low speed limit(KB/s) |
287 | | DECLARE_mInt32(download_low_speed_limit_kbps); |
288 | | // download low speed time(seconds) |
289 | | DECLARE_mInt32(download_low_speed_time); |
290 | | |
291 | | // deprecated, use env var LOG_DIR in be.conf |
292 | | DECLARE_String(sys_log_dir); |
293 | | // for udf |
294 | | DECLARE_String(user_function_dir); |
295 | | // INFO, WARNING, ERROR, FATAL |
296 | | DECLARE_String(sys_log_level); |
297 | | // TIME-DAY, TIME-HOUR, SIZE-MB-nnn |
298 | | DECLARE_String(sys_log_roll_mode); |
299 | | // log roll num |
300 | | DECLARE_Int32(sys_log_roll_num); |
301 | | // verbose log |
302 | | DECLARE_Strings(sys_log_verbose_modules); |
303 | | // verbose log level |
304 | | DECLARE_Int32(sys_log_verbose_level); |
305 | | // verbose log FLAGS_v |
306 | | DECLARE_Int32(sys_log_verbose_flags_v); |
307 | | // log buffer level |
308 | | DECLARE_String(log_buffer_level); |
309 | | // log enable custom date time format |
310 | | DECLARE_Bool(sys_log_enable_custom_date_time_format); |
311 | | // log custom date time format (https://en.cppreference.com/w/cpp/io/manip/put_time) |
312 | | DECLARE_String(sys_log_custom_date_time_format); |
313 | | // log custom date time milliseconds format (fmt::format) |
314 | | DECLARE_String(sys_log_custom_date_time_ms_format); |
315 | | |
316 | | // number of threads available to serve backend execution requests |
317 | | DECLARE_Int32(be_service_threads); |
318 | | |
319 | | // interval between profile reports; in seconds |
320 | | DECLARE_mInt32(status_report_interval); |
321 | | DECLARE_mInt32(pipeline_status_report_interval); |
322 | | // if true, each disk will have a separate thread pool for scanner |
323 | | DECLARE_Bool(doris_enable_scanner_thread_pool_per_disk); |
324 | | // the timeout of a work thread to wait the blocking priority queue to get a task |
325 | | DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms); |
326 | | // number of scanner thread pool size for olap table |
327 | | // and the min thread num of remote scanner thread pool |
328 | | DECLARE_mInt32(doris_scanner_thread_pool_thread_num); |
329 | | DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num); |
330 | | // number of batch size to fetch the remote split source |
331 | | DECLARE_mInt32(remote_split_source_batch_size); |
332 | | // max number of remote scanner thread pool size |
333 | | // if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10) |
334 | | DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num); |
335 | | // number of olap scanner thread pool queue size |
336 | | DECLARE_Int32(doris_scanner_thread_pool_queue_size); |
337 | | // default thrift client connect timeout(in seconds) |
338 | | DECLARE_mInt32(thrift_connect_timeout_seconds); |
339 | | DECLARE_mInt32(fetch_rpc_timeout_seconds); |
340 | | // default thrift client retry interval (in milliseconds) |
341 | | DECLARE_mInt64(thrift_client_retry_interval_ms); |
342 | | // max message size of thrift request |
343 | | // default: 100 * 1024 * 1024 |
344 | | DECLARE_mInt64(thrift_max_message_size); |
345 | | // max row count number for single scan range, used in segmentv1 |
346 | | DECLARE_mInt32(doris_scan_range_row_count); |
347 | | // max bytes number for single scan range, used in segmentv2 |
348 | | DECLARE_mInt32(doris_scan_range_max_mb); |
349 | | // max bytes number for single scan block, used in segmentv2 |
350 | | DECLARE_mInt32(doris_scan_block_max_mb); |
351 | | // size of scanner queue between scanner thread and compute thread |
352 | | DECLARE_mInt32(doris_scanner_queue_size); |
353 | | // single read execute fragment row number |
354 | | DECLARE_mInt32(doris_scanner_row_num); |
355 | | // single read execute fragment row bytes |
356 | | DECLARE_mInt32(doris_scanner_row_bytes); |
357 | | // single read execute fragment max run time millseconds |
358 | | DECLARE_mInt32(doris_scanner_max_run_time_ms); |
359 | | DECLARE_mInt32(min_bytes_in_scanner_queue); |
360 | | // number of max scan keys |
361 | | DECLARE_mInt32(doris_max_scan_key_num); |
362 | | // the max number of push down values of a single column. |
363 | | // if exceed, no conditions will be pushed down for that column. |
364 | | DECLARE_mInt32(max_pushdown_conditions_per_column); |
365 | | // (Advanced) Maximum size of per-query receive-side buffer |
366 | | DECLARE_mInt32(exchg_node_buffer_size_bytes); |
367 | | DECLARE_mInt32(exchg_buffer_queue_capacity_factor); |
368 | | |
369 | | DECLARE_mInt64(column_dictionary_key_ratio_threshold); |
370 | | DECLARE_mInt64(column_dictionary_key_size_threshold); |
371 | | // memory_limitation_per_thread_for_schema_change_bytes unit bytes |
372 | | DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes); |
373 | | DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes); |
374 | | |
375 | | // all cache prune interval, used by GC and periodic thread. |
376 | | DECLARE_mInt32(cache_prune_interval_sec); |
377 | | DECLARE_mInt32(cache_periodic_prune_stale_sweep_sec); |
378 | | // the clean interval of tablet lookup cache |
379 | | DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec); |
380 | | DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec); |
381 | | DECLARE_mInt32(disk_stat_monitor_interval); |
382 | | DECLARE_mInt32(unused_rowset_monitor_interval); |
383 | | DECLARE_String(storage_root_path); |
384 | | DECLARE_mString(broken_storage_path); |
385 | | |
386 | | // Config is used to check incompatible old format hdr_ format |
387 | | // whether doris uses strict way. When config is true, process will log fatal |
388 | | // and exit. When config is false, process will only log warning. |
389 | | DECLARE_Bool(storage_strict_check_incompatible_old_format); |
390 | | |
391 | | // BE process will exit if the percentage of error disk reach this value. |
392 | | DECLARE_mInt32(max_percentage_of_error_disk); |
393 | | DECLARE_mInt32(default_num_rows_per_column_file_block); |
394 | | // pending data policy |
395 | | DECLARE_mInt32(pending_data_expire_time_sec); |
396 | | // inc_rowset snapshot rs sweep time interval |
397 | | DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec); |
398 | | // tablet stale rowset sweep by threshold size |
399 | | DECLARE_Bool(tablet_rowset_stale_sweep_by_size); |
400 | | DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size); |
401 | | // garbage sweep policy |
402 | | DECLARE_Int32(max_garbage_sweep_interval); |
403 | | DECLARE_Int32(min_garbage_sweep_interval); |
404 | | // garbage sweep every batch will sleep 1ms |
405 | | DECLARE_mInt32(garbage_sweep_batch_size); |
406 | | DECLARE_mInt32(snapshot_expire_time_sec); |
407 | | // It is only a recommended value. When the disk space is insufficient, |
408 | | // the file storage period under trash dose not have to comply with this parameter. |
409 | | DECLARE_mInt32(trash_file_expire_time_sec); |
410 | | // minimum file descriptor number |
411 | | // modify them upon necessity |
412 | | DECLARE_Int32(min_file_descriptor_number); |
413 | | DECLARE_mBool(disable_segment_cache); |
414 | | DECLARE_String(row_cache_mem_limit); |
415 | | |
416 | | // Cache for storage page size |
417 | | DECLARE_String(storage_page_cache_limit); |
418 | | // Shard size for page cache, the value must be power of two. |
419 | | // It's recommended to set it to a value close to the number of BE cores in order to reduce lock contentions. |
420 | | DECLARE_Int32(storage_page_cache_shard_size); |
421 | | // Percentage for index page cache |
422 | | // all storage page cache will be divided into data_page_cache and index_page_cache |
423 | | DECLARE_Int32(index_page_cache_percentage); |
424 | | // whether to disable page cache feature in storage |
425 | | // TODO delete it. Divided into Data page, Index page, pk index page |
426 | | DECLARE_Bool(disable_storage_page_cache); |
427 | | // whether to disable row cache feature in storage |
428 | | DECLARE_mBool(disable_storage_row_cache); |
429 | | // whether to disable pk page cache feature in storage |
430 | | DECLARE_Bool(disable_pk_storage_page_cache); |
431 | | |
432 | | // Cache for mow primary key storage page size, it's seperated from |
433 | | // storage_page_cache_limit |
434 | | DECLARE_String(pk_storage_page_cache_limit); |
435 | | // data page size for primary key index |
436 | | DECLARE_Int32(primary_key_data_page_size); |
437 | | |
438 | | // inc_rowset snapshot rs sweep time interval |
439 | | DECLARE_mInt32(data_page_cache_stale_sweep_time_sec); |
440 | | DECLARE_mInt32(index_page_cache_stale_sweep_time_sec); |
441 | | // great impact on the performance of MOW, so it can be longer. |
442 | | DECLARE_mInt32(pk_index_page_cache_stale_sweep_time_sec); |
443 | | |
444 | | DECLARE_Bool(enable_low_cardinality_optimize); |
445 | | DECLARE_Bool(enable_low_cardinality_cache_code); |
446 | | |
447 | | // be policy |
448 | | // whether check compaction checksum |
449 | | DECLARE_mBool(enable_compaction_checksum); |
450 | | // whether disable automatic compaction task |
451 | | DECLARE_mBool(disable_auto_compaction); |
452 | | // whether enable vertical compaction |
453 | | DECLARE_mBool(enable_vertical_compaction); |
454 | | // whether enable ordered data compaction |
455 | | DECLARE_mBool(enable_ordered_data_compaction); |
456 | | // In vertical compaction, column number for every group |
457 | | DECLARE_mInt32(vertical_compaction_num_columns_per_group); |
458 | | // In vertical compaction, max memory usage for row_source_buffer |
459 | | DECLARE_Int32(vertical_compaction_max_row_source_memory_mb); |
460 | | // In vertical compaction, max dest segment file size |
461 | | DECLARE_mInt64(vertical_compaction_max_segment_size); |
462 | | |
463 | | // If enabled, segments will be flushed column by column |
464 | | DECLARE_mBool(enable_vertical_segment_writer); |
465 | | |
466 | | // In ordered data compaction, min segment size for input rowset |
467 | | DECLARE_mInt32(ordered_data_compaction_min_segment_size); |
468 | | |
469 | | // This config can be set to limit thread number in compaction thread pool. |
470 | | DECLARE_mInt32(max_base_compaction_threads); |
471 | | DECLARE_mInt32(max_cumu_compaction_threads); |
472 | | DECLARE_mInt32(max_single_replica_compaction_threads); |
473 | | |
474 | | DECLARE_Bool(enable_base_compaction_idle_sched); |
475 | | DECLARE_mInt64(base_compaction_min_rowset_num); |
476 | | DECLARE_mInt64(base_compaction_max_compaction_score); |
477 | | DECLARE_mDouble(base_compaction_min_data_ratio); |
478 | | DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes); |
479 | | |
480 | | DECLARE_Bool(enable_skip_tablet_compaction); |
481 | | // output rowset of cumulative compaction total disk size exceed this config size, |
482 | | // this rowset will be given to base compaction, unit is m byte. |
483 | | DECLARE_mInt64(compaction_promotion_size_mbytes); |
484 | | |
485 | | // output rowset of cumulative compaction total disk size exceed this config ratio of |
486 | | // base rowset's total disk size, this rowset will be given to base compaction. The value must be between |
487 | | // 0 and 1. |
488 | | DECLARE_mDouble(compaction_promotion_ratio); |
489 | | |
490 | | // the smallest size of rowset promotion. When the rowset is less than this config, this |
491 | | // rowset will be not given to base compaction. The unit is m byte. |
492 | | DECLARE_mInt64(compaction_promotion_min_size_mbytes); |
493 | | |
494 | | // When output rowset of cumulative compaction total version count (end_version - start_version) |
495 | | // exceed this config count, the rowset will be moved to base compaction |
496 | | // NOTE: this config will work for unique key merge-on-write table only, to reduce version count |
497 | | // related cost on delete bitmap more effectively. |
498 | | DECLARE_mInt64(compaction_promotion_version_count); |
499 | | |
500 | | // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than |
501 | | // this size, size_based policy may not do to cumulative compaction. The unit is m byte. |
502 | | DECLARE_mInt64(compaction_min_size_mbytes); |
503 | | |
504 | | // cumulative compaction policy: min and max delta file's number |
505 | | DECLARE_mInt64(cumulative_compaction_min_deltas); |
506 | | DECLARE_mInt64(cumulative_compaction_max_deltas); |
507 | | DECLARE_mInt32(cumulative_compaction_max_deltas_factor); |
508 | | |
509 | | // This config can be set to limit thread number in multiget thread pool. |
510 | | DECLARE_mInt32(multi_get_max_threads); |
511 | | |
512 | | // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. |
513 | | DECLARE_mInt64(total_permits_for_compaction_score); |
514 | | |
515 | | // sleep interval in ms after generated compaction tasks |
516 | | DECLARE_mInt32(generate_compaction_tasks_interval_ms); |
517 | | // sleep interval in second after update replica infos |
518 | | DECLARE_mInt32(update_replica_infos_interval_seconds); |
519 | | |
520 | | // Compaction task number per disk. |
521 | | // Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each. |
522 | | DECLARE_mInt32(compaction_task_num_per_disk); |
523 | | // compaction thread num for fast disk(typically .SSD), must be greater than 2. |
524 | | DECLARE_mInt32(compaction_task_num_per_fast_disk); |
525 | | |
526 | | // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. |
527 | | DECLARE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round); |
528 | | |
529 | | // Not compact the invisible versions, but with some limitations: |
530 | | // if not timeout, keep no more than compaction_keep_invisible_version_max_count versions; |
531 | | // if timeout, keep no more than compaction_keep_invisible_version_min_count versions. |
532 | | DECLARE_mInt32(compaction_keep_invisible_version_timeout_sec); |
533 | | DECLARE_mInt32(compaction_keep_invisible_version_min_count); |
534 | | DECLARE_mInt32(compaction_keep_invisible_version_max_count); |
535 | | |
536 | | // Threshold to logging compaction trace, in seconds. |
537 | | DECLARE_mInt32(base_compaction_trace_threshold); |
538 | | DECLARE_mInt32(cumulative_compaction_trace_threshold); |
539 | | DECLARE_mBool(disable_compaction_trace_log); |
540 | | |
541 | | // Interval to picking rowset to compact, in seconds |
542 | | DECLARE_mInt64(pick_rowset_to_compact_interval_sec); |
543 | | |
544 | | // Compaction priority schedule |
545 | | DECLARE_mBool(enable_compaction_priority_scheduling); |
546 | | DECLARE_mInt32(low_priority_compaction_task_num_per_disk); |
547 | | DECLARE_mInt32(low_priority_compaction_score_threshold); |
548 | | |
549 | | // Thread count to do tablet meta checkpoint, -1 means use the data directories count. |
550 | | DECLARE_Int32(max_meta_checkpoint_threads); |
551 | | |
552 | | // Threshold to logging agent task trace, in seconds. |
553 | | DECLARE_mInt32(agent_task_trace_threshold_sec); |
554 | | |
555 | | // This config can be set to limit thread number in tablet migration thread pool. |
556 | | DECLARE_Int32(min_tablet_migration_threads); |
557 | | DECLARE_Int32(max_tablet_migration_threads); |
558 | | |
559 | | DECLARE_mInt32(finished_migration_tasks_size); |
560 | | // If size less than this, the remaining rowsets will be force to complete |
561 | | DECLARE_mInt32(migration_remaining_size_threshold_mb); |
562 | | // If the task runs longer than this time, the task will be terminated, in seconds. |
563 | | // timeout = std::max(migration_task_timeout_secs, tablet size / 1MB/s) |
564 | | DECLARE_mInt32(migration_task_timeout_secs); |
565 | | // timeout for try_lock migration lock |
566 | | DECLARE_Int64(migration_lock_timeout_ms); |
567 | | |
568 | | // Port to start debug webserver on |
569 | | DECLARE_Int32(webserver_port); |
570 | | // Https enable flag |
571 | | DECLARE_Bool(enable_https); |
572 | | // Path of certificate |
573 | | DECLARE_String(ssl_certificate_path); |
574 | | // Path of private key |
575 | | DECLARE_String(ssl_private_key_path); |
576 | | // Whether to check authorization |
577 | | DECLARE_Bool(enable_all_http_auth); |
578 | | // Number of webserver workers |
579 | | DECLARE_Int32(webserver_num_workers); |
580 | | |
581 | | DECLARE_Bool(enable_single_replica_load); |
582 | | // Number of download workers for single replica load |
583 | | DECLARE_Int32(single_replica_load_download_num_workers); |
584 | | |
585 | | // Used for mini Load. mini load data file will be removed after this time. |
586 | | DECLARE_Int64(load_data_reserve_hours); |
587 | | // log error log will be removed after this time |
588 | | DECLARE_mInt64(load_error_log_reserve_hours); |
589 | | // error log size limit, default 200MB |
590 | | DECLARE_mInt64(load_error_log_limit_bytes); |
591 | | |
592 | | // be brpc interface is classified into two categories: light and heavy |
593 | | // each category has diffrent thread number |
594 | | // threads to handle heavy api interface, such as transmit_data/transmit_block etc |
595 | | // Default, if less than or equal 32 core, the following are 128, 128, 10240, 10240 in turn. |
596 | | // if greater than 32 core, the following are core num * 4, core num * 4, core num * 320, core num * 320 in turn |
597 | | DECLARE_Int32(brpc_heavy_work_pool_threads); |
598 | | // threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start |
599 | | DECLARE_Int32(brpc_light_work_pool_threads); |
600 | | DECLARE_Int32(brpc_heavy_work_pool_max_queue_size); |
601 | | DECLARE_Int32(brpc_light_work_pool_max_queue_size); |
602 | | DECLARE_Int32(brpc_arrow_flight_work_pool_threads); |
603 | | DECLARE_Int32(brpc_arrow_flight_work_pool_max_queue_size); |
604 | | |
605 | | // The maximum amount of data that can be processed by a stream load |
606 | | DECLARE_mInt64(streaming_load_max_mb); |
607 | | // Some data formats, such as JSON, cannot be streamed. |
608 | | // Therefore, it is necessary to limit the maximum number of |
609 | | // such data when using stream load to prevent excessive memory consumption. |
610 | | DECLARE_mInt64(streaming_load_json_max_mb); |
611 | | // the alive time of a TabletsChannel. |
612 | | // If the channel does not receive any data till this time, |
613 | | // the channel will be removed. |
614 | | DECLARE_mInt32(streaming_load_rpc_max_alive_time_sec); |
615 | | // the timeout of a rpc to open the tablet writer in remote BE. |
616 | | // short operation time, can set a short timeout |
617 | | DECLARE_Int32(tablet_writer_open_rpc_timeout_sec); |
618 | | // You can ignore brpc error '[E1011]The server is overcrowded' when writing data. |
619 | | DECLARE_mBool(tablet_writer_ignore_eovercrowded); |
620 | | DECLARE_mBool(exchange_sink_ignore_eovercrowded); |
621 | | DECLARE_mInt32(slave_replica_writer_rpc_timeout_sec); |
622 | | // Whether to enable stream load record function, the default is false. |
623 | | // False: disable stream load record |
624 | | DECLARE_mBool(enable_stream_load_record); |
625 | | // batch size of stream load record reported to FE |
626 | | DECLARE_mInt32(stream_load_record_batch_size); |
627 | | // expire time of stream load record in rocksdb. |
628 | | DECLARE_Int32(stream_load_record_expire_time_secs); |
629 | | // time interval to clean expired stream load records |
630 | | DECLARE_mInt64(clean_stream_load_record_interval_secs); |
631 | | // The buffer size to store stream table function schema info |
632 | | DECLARE_Int64(stream_tvf_buffer_size); |
633 | | |
634 | | // OlapTableSink sender's send interval, should be less than the real response time of a tablet writer rpc. |
635 | | // You may need to lower the speed when the sink receiver bes are too busy. |
636 | | DECLARE_mInt32(olap_table_sink_send_interval_microseconds); |
637 | | // For auto partition, the send interval will multiply the factor |
638 | | DECLARE_mDouble(olap_table_sink_send_interval_auto_partition_factor); |
639 | | |
640 | | // Fragment thread pool |
641 | | DECLARE_Int32(fragment_pool_thread_num_min); |
642 | | DECLARE_Int32(fragment_pool_thread_num_max); |
643 | | DECLARE_Int32(fragment_pool_queue_size); |
644 | | |
645 | | // Control the number of disks on the machine. If 0, this comes from the system settings. |
646 | | DECLARE_Int32(num_disks); |
647 | | // The maximum number of the threads per disk is also the max queue depth per disk. |
648 | | DECLARE_Int32(num_threads_per_disk); |
649 | | // The read size is the size of the reads sent to os. |
650 | | // There is a trade off of latency and throughout, trying to keep disks busy but |
651 | | // not introduce seeks. The literature seems to agree that with 8 MB reads, random |
652 | | // io and sequential io perform similarly. |
653 | | DECLARE_Int32(read_size); // 8 * 1024 * 1024, Read Size (in bytes) |
654 | | DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes) |
655 | | |
656 | | // for pprof |
657 | | DECLARE_String(pprof_profile_dir); |
658 | | // for jeprofile in jemalloc |
659 | | DECLARE_mString(jeprofile_dir); |
660 | | // Purge all unused dirty pages for all arenas. |
661 | | DECLARE_mBool(enable_je_purge_dirty_pages); |
662 | | // Purge all unused Jemalloc dirty pages for all arenas when exceed je_dirty_pages_mem_limit and process exceed soft limit. |
663 | | DECLARE_mString(je_dirty_pages_mem_limit_percent); |
664 | | |
665 | | // to forward compatibility, will be removed later |
666 | | DECLARE_mBool(enable_token_check); |
667 | | |
668 | | // to open/close system metrics |
669 | | DECLARE_Bool(enable_system_metrics); |
670 | | |
671 | | // Number of cores Doris will used, this will effect only when it's greater than 0. |
672 | | // Otherwise, Doris will use all cores returned from "/proc/cpuinfo". |
673 | | DECLARE_Int32(num_cores); |
674 | | |
675 | | // When BE start, If there is a broken disk, BE process will exit by default. |
676 | | // Otherwise, we will ignore the broken disk, |
677 | | DECLARE_Bool(ignore_broken_disk); |
678 | | |
679 | | // Sleep time in milliseconds between memory maintenance iterations |
680 | | DECLARE_mInt32(memory_maintenance_sleep_time_ms); |
681 | | |
682 | | // After full gc, no longer full gc and minor gc during sleep. |
683 | | // After minor gc, no minor gc during sleep, but full gc is possible. |
684 | | DECLARE_mInt32(memory_gc_sleep_time_ms); |
685 | | |
686 | | // Sleep time in milliseconds between memtbale flush mgr memory refresh iterations |
687 | | DECLARE_mInt64(memtable_mem_tracker_refresh_interval_ms); |
688 | | |
689 | | // Sleep time in milliseconds between refresh iterations of workload group weighted memory ratio |
690 | | DECLARE_mInt64(wg_weighted_memory_ratio_refresh_interval_ms); |
691 | | |
692 | | // percent of (active memtables size / all memtables size) when reach hard limit |
693 | | DECLARE_mInt32(memtable_hard_limit_active_percent); |
694 | | |
695 | | // percent of (active memtables size / all memtables size) when reach soft limit |
696 | | DECLARE_mInt32(memtable_soft_limit_active_percent); |
697 | | |
698 | | // memtable insert memory tracker will multiply input block size with this ratio |
699 | | DECLARE_mDouble(memtable_insert_memory_ratio); |
700 | | // max write buffer size before flush, default 200MB |
701 | | DECLARE_mInt64(write_buffer_size); |
702 | | // max buffer size used in memtable for the aggregated table, default 400MB |
703 | | DECLARE_mInt64(write_buffer_size_for_agg); |
704 | | // max parallel flush task per memtable writer |
705 | | DECLARE_mInt32(memtable_flush_running_count_limit); |
706 | | |
707 | | DECLARE_Int32(load_process_max_memory_limit_percent); // 50% |
708 | | |
709 | | // If the memory consumption of load jobs exceed load_process_max_memory_limit, |
710 | | // all load jobs will hang there to wait for memtable flush. We should have a |
711 | | // soft limit which can trigger the memtable flush for the load channel who |
712 | | // consumes lagest memory size before we reach the hard limit. The soft limit |
713 | | // might avoid all load jobs hang at the same time. |
714 | | DECLARE_Int32(load_process_soft_mem_limit_percent); |
715 | | |
716 | | // If load memory consumption is within load_process_safe_mem_permit_percent, |
717 | | // memtable memory limiter will do nothing. |
718 | | DECLARE_Int32(load_process_safe_mem_permit_percent); |
719 | | |
720 | | // result buffer cancelled time (unit: second) |
721 | | DECLARE_mInt32(result_buffer_cancelled_interval_time); |
722 | | |
723 | | // arrow flight result sink buffer rows size, default 4096 * 8 |
724 | | DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows); |
725 | | // The timeout for ADBC Client to wait for data using arrow flight reader. |
726 | | // If the query is very complex and no result is generated after this time, consider increasing this timeout. |
727 | | DECLARE_mInt32(arrow_flight_reader_brpc_controller_timeout_ms); |
728 | | |
729 | | // the increased frequency of priority for remaining tasks in BlockingPriorityQueue |
730 | | DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency); |
731 | | |
732 | | // sync tablet_meta when modifying meta |
733 | | DECLARE_mBool(sync_tablet_meta); |
734 | | |
735 | | // default thrift rpc timeout ms |
736 | | DECLARE_mInt32(thrift_rpc_timeout_ms); |
737 | | |
738 | | // txn commit rpc timeout |
739 | | DECLARE_mInt32(txn_commit_rpc_timeout_ms); |
740 | | |
741 | | // If set to true, metric calculator will run |
742 | | DECLARE_Bool(enable_metric_calculator); |
743 | | |
744 | | // max consumer num in one data consumer group, for routine load |
745 | | DECLARE_mInt32(max_consumer_num_per_group); |
746 | | |
747 | | // the max size of thread pool for routine load task. |
748 | | // this should be larger than FE config 'max_routine_load_task_num_per_be' (default 5) |
749 | | DECLARE_Int32(max_routine_load_thread_pool_size); |
750 | | |
751 | | // max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row |
752 | | // default is 20, batch_size's default value is 1024 means 20 * 1024 rows will be cached |
753 | | DECLARE_mInt32(max_memory_sink_batch_count); |
754 | | |
755 | | // This configuration is used for the context gc thread schedule period |
756 | | // note: unit is minute, default is 5min |
757 | | DECLARE_mInt32(scan_context_gc_interval_min); |
758 | | |
759 | | // es scroll keep-alive |
760 | | DECLARE_String(es_scroll_keepalive); |
761 | | |
762 | | // HTTP connection timeout for es |
763 | | DECLARE_mInt32(es_http_timeout_ms); |
764 | | |
765 | | // the max client cache number per each host |
766 | | // There are variety of client cache in BE, but currently we use the |
767 | | // same cache size configuration. |
768 | | // TODO(cmy): use different config to set different client cache if necessary. |
769 | | DECLARE_Int32(max_client_cache_size_per_host); |
770 | | |
771 | | // Dir to save files downloaded by SmallFileMgr |
772 | | DECLARE_String(small_file_dir); |
773 | | // path gc |
774 | | DECLARE_Bool(path_gc_check); |
775 | | DECLARE_mInt32(path_gc_check_interval_second); |
776 | | DECLARE_mInt32(path_gc_check_step); |
777 | | DECLARE_mInt32(path_gc_check_step_interval_ms); |
778 | | |
779 | | // The following 2 configs limit the max usage of disk capacity of a data dir. |
780 | | // If both of these 2 threshold reached, no more data can be writen into that data dir. |
781 | | // The percent of max used capacity of a data dir |
782 | | DECLARE_mInt32(storage_flood_stage_usage_percent); // 90% |
783 | | // The min bytes that should be left of a data dir |
784 | | DECLARE_mInt64(storage_flood_stage_left_capacity_bytes); // 1GB |
785 | | // number of thread for flushing memtable per store |
786 | | DECLARE_Int32(flush_thread_num_per_store); |
787 | | // number of thread for flushing memtable per store, for high priority load task |
788 | | DECLARE_Int32(high_priority_flush_thread_num_per_store); |
789 | | |
790 | | // workload group's flush thread num |
791 | | DECLARE_Int32(wg_flush_thread_num_per_store); |
792 | | |
793 | | // config for tablet meta checkpoint |
794 | | DECLARE_mInt32(tablet_meta_checkpoint_min_new_rowsets_num); |
795 | | DECLARE_mInt32(tablet_meta_checkpoint_min_interval_secs); |
796 | | DECLARE_Int32(generate_tablet_meta_checkpoint_tasks_interval_secs); |
797 | | |
798 | | // config for default rowset type |
799 | | // Valid configs: ALPHA, BETA |
800 | | DECLARE_String(default_rowset_type); |
801 | | |
802 | | // Maximum size of a single message body in all protocols |
803 | | DECLARE_Int64(brpc_max_body_size); |
804 | | // Max unwritten bytes in each socket, if the limit is reached, Socket.Write fails with EOVERCROWDED |
805 | | // Default, if the physical memory is less than or equal to 64G, the value is 1G |
806 | | // if the physical memory is greater than 64G, the value is physical memory * mem_limit(0.8) / 1024 * 20 |
807 | | DECLARE_Int64(brpc_socket_max_unwritten_bytes); |
808 | | // TODO(zxy): expect to be true in v1.3 |
809 | | // Whether to embed the ProtoBuf Request serialized string together with Tuple/Block data into |
810 | | // Controller Attachment and send it through http brpc when the length of the Tuple/Block data |
811 | | // is greater than 1.8G. This is to avoid the error of Request length overflow (2G). |
812 | | DECLARE_mBool(transfer_large_data_by_brpc); |
813 | | |
814 | | // max number of txns for every txn_partition_map in txn manager |
815 | | // this is a self protection to avoid too many txns saving in manager |
816 | | DECLARE_mInt64(max_runnings_transactions_per_txn_map); |
817 | | |
818 | | // tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4 |
819 | | // this is a an enhancement for better performance to manage tablet |
820 | | DECLARE_Int32(tablet_map_shard_size); |
821 | | |
822 | | // txn_map_lock shard size, the value is 2^n, n=0,1,2,3,4 |
823 | | // this is a an enhancement for better performance to manage txn |
824 | | DECLARE_Int32(txn_map_shard_size); |
825 | | |
826 | | // txn_lock shard size, the value is 2^n, n=0,1,2,3,4 |
827 | | // this is a an enhancement for better performance to commit and publish txn |
828 | | DECLARE_Int32(txn_shard_size); |
829 | | |
830 | | // Whether to continue to start be when load tablet from header failed. |
831 | | DECLARE_Bool(ignore_load_tablet_failure); |
832 | | |
833 | | // Whether to continue to start be when load tablet from header failed. |
834 | | DECLARE_mBool(ignore_rowset_stale_unconsistent_delete); |
835 | | |
836 | | // Set max cache's size of query results, the unit is M byte |
837 | | DECLARE_Int32(query_cache_max_size_mb); |
838 | | |
839 | | // Cache memory is pruned when reach query_cache_max_size_mb + query_cache_elasticity_size_mb |
840 | | DECLARE_Int32(query_cache_elasticity_size_mb); |
841 | | |
842 | | // Maximum number of cache partitions corresponding to a SQL |
843 | | DECLARE_Int32(query_cache_max_partition_count); |
844 | | |
845 | | // Maximum number of version of a tablet. If the version num of a tablet exceed limit, |
846 | | // the load process will reject new incoming load job of this tablet. |
847 | | // This is to avoid too many version num. |
848 | | DECLARE_mInt32(max_tablet_version_num); |
849 | | |
850 | | // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server, |
851 | | // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport |
852 | | DECLARE_String(thrift_server_type_of_fe); |
853 | | |
854 | | // disable zone map index when page row is too few |
855 | | DECLARE_mInt32(zone_map_row_num_threshold); |
856 | | |
857 | | // aws sdk log level |
858 | | // Off = 0, |
859 | | // Fatal = 1, |
860 | | // Error = 2, |
861 | | // Warn = 3, |
862 | | // Info = 4, |
863 | | // Debug = 5, |
864 | | // Trace = 6 |
865 | | DECLARE_Int32(aws_log_level); |
866 | | |
867 | | // the buffer size when read data from remote storage like s3 |
868 | | DECLARE_mInt32(remote_storage_read_buffer_mb); |
869 | | |
870 | | // The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size |
871 | | // smaller than this value will continue to accumulate. specified as number of bytes. |
872 | | // Decreasing this value will increase the frequency of consume/release. |
873 | | // Increasing this value will cause MemTracker statistics to be inaccurate. |
874 | | DECLARE_mInt32(mem_tracker_consume_min_size_bytes); |
875 | | |
876 | | // The version information of the tablet will be stored in the memory |
877 | | // in an adjacency graph data structure. |
878 | | // And as the new version is written and the old version is deleted, |
879 | | // the data structure will begin to have empty vertex with no edge associations(orphan vertex). |
880 | | // This config is used to control that when the proportion of orphan vertex is greater than the threshold, |
881 | | // the adjacency graph will be rebuilt to ensure that the data structure will not expand indefinitely. |
882 | | // This config usually only needs to be modified during testing. |
883 | | // In most cases, it does not need to be modified. |
884 | | DECLARE_mDouble(tablet_version_graph_orphan_vertex_ratio); |
885 | | |
886 | | // share delta writers when memtable_on_sink_node = true |
887 | | DECLARE_Bool(share_delta_writers); |
888 | | // timeout for open load stream rpc in ms |
889 | | DECLARE_Int64(open_load_stream_timeout_ms); |
890 | | // enable write background when using brpc stream |
891 | | DECLARE_mBool(enable_brpc_stream_write_background); |
892 | | |
893 | | // brpc streaming max_buf_size in bytes |
894 | | DECLARE_Int64(load_stream_max_buf_size); |
895 | | // brpc streaming messages_in_batch |
896 | | DECLARE_Int32(load_stream_messages_in_batch); |
897 | | // brpc streaming StreamWait seconds on EAGAIN |
898 | | DECLARE_Int32(load_stream_eagain_wait_seconds); |
899 | | // max tasks per flush token in load stream |
900 | | DECLARE_Int32(load_stream_flush_token_max_tasks); |
901 | | // max wait flush token time in load stream |
902 | | DECLARE_Int32(load_stream_max_wait_flush_token_time_ms); |
903 | | |
904 | | // max send batch parallelism for OlapTableSink |
905 | | // The value set by the user for send_batch_parallelism is not allowed to exceed max_send_batch_parallelism_per_job, |
906 | | // if exceed, the value of send_batch_parallelism would be max_send_batch_parallelism_per_job |
907 | | DECLARE_mInt32(max_send_batch_parallelism_per_job); |
908 | | |
909 | | // number of send batch thread pool size |
910 | | DECLARE_Int32(send_batch_thread_pool_thread_num); |
911 | | // number of send batch thread pool queue size |
912 | | DECLARE_Int32(send_batch_thread_pool_queue_size); |
913 | | |
914 | | // Limit the number of segment of a newly created rowset. |
915 | | // The newly created rowset may to be compacted after loading, |
916 | | // so if there are too many segment in a rowset, the compaction process |
917 | | // will run out of memory. |
918 | | // When doing compaction, each segment may take at least 1MB buffer. |
919 | | DECLARE_mInt32(max_segment_num_per_rowset); |
920 | | |
921 | | // Store segment without compression if a segment is smaller than |
922 | | // segment_compression_threshold_kb. |
923 | | DECLARE_mInt32(segment_compression_threshold_kb); |
924 | | |
925 | | // The connection timeout when connecting to external table such as odbc table. |
926 | | DECLARE_mInt32(external_table_connect_timeout_sec); |
927 | | |
928 | | // Time to clean up useless JDBC connection pool cache |
929 | | DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec); |
930 | | |
931 | | // Global bitmap cache capacity for aggregation cache, size in bytes |
932 | | DECLARE_Int64(delete_bitmap_agg_cache_capacity); |
933 | | DECLARE_String(delete_bitmap_dynamic_agg_cache_limit); |
934 | | DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec); |
935 | | |
936 | | // A common object cache depends on an Sharded LRU Cache. |
937 | | DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec); |
938 | | |
939 | | // reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility |
940 | | // If the dependent kafka broker version older than 0.10.0.0, |
941 | | // the value of kafka_api_version_request should be false, and the |
942 | | // value set by the fallback version kafka_broker_version_fallback will be used, |
943 | | // and the valid values are: 0.9.0.x, 0.8.x.y. |
944 | | DECLARE_String(kafka_api_version_request); |
945 | | DECLARE_String(kafka_broker_version_fallback); |
946 | | DECLARE_mString(kafka_debug); |
947 | | |
948 | | // The number of pool siz of routine load consumer. |
949 | | // If you meet the error describe in https://github.com/edenhill/librdkafka/issues/3608 |
950 | | // Change this size to 0 to fix it temporarily. |
951 | | DECLARE_mInt32(routine_load_consumer_pool_size); |
952 | | |
953 | | // Used in single-stream-multi-table load. When receive a batch of messages from kafka, |
954 | | // if the size of batch is more than this threshold, we will request plans for all related tables. |
955 | | DECLARE_Int32(multi_table_batch_plan_threshold); |
956 | | |
957 | | // Used in single-stream-multi-table load. When receiving a batch of messages from Kafka, |
958 | | // if the size of the table wait for plan is more than this threshold, we will request plans for all related tables. |
959 | | // The param is aimed to avoid requesting and executing too many plans at once. |
960 | | // Performing small batch processing on multiple tables during the loaded process can reduce the pressure of a single RPC |
961 | | // and improve the real-time processing of data. |
962 | | DECLARE_Int32(multi_table_max_wait_tables); |
963 | | |
964 | | // When the timeout of a load task is less than this threshold, |
965 | | // Doris treats it as a high priority task. |
966 | | // high priority tasks use a separate thread pool for flush and do not block rpc by memory cleanup logic. |
967 | | // this threshold is mainly used to identify routine load tasks and should not be modified if not necessary. |
968 | | DECLARE_mInt32(load_task_high_priority_threshold_second); |
969 | | |
970 | | // The min timeout of load rpc (add batch, close) |
971 | | // Because a load rpc may be blocked for a while. |
972 | | // Increase this config may avoid rpc timeout. |
973 | | DECLARE_mInt32(min_load_rpc_timeout_ms); |
974 | | |
975 | | // use which protocol to access function service, candicate is baidu_std/h2:grpc |
976 | | DECLARE_String(function_service_protocol); |
977 | | |
978 | | // use which load balancer to select server to connect |
979 | | DECLARE_String(rpc_load_balancer); |
980 | | |
981 | | // a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance, |
982 | | // so we set a soft limit, default is 1MB |
983 | | DECLARE_mInt32(string_type_length_soft_limit_bytes); |
984 | | |
985 | | DECLARE_mInt32(jsonb_type_length_soft_limit_bytes); |
986 | | |
987 | | // Threshold fo reading a small file into memory |
988 | | DECLARE_mInt32(in_memory_file_size); |
989 | | |
990 | | // ParquetReaderWrap prefetch buffer size |
991 | | DECLARE_Int32(parquet_reader_max_buffer_size); |
992 | | // Max size of parquet page header in bytes |
993 | | DECLARE_mInt32(parquet_header_max_size_mb); |
994 | | // Max buffer size for parquet row group |
995 | | DECLARE_mInt32(parquet_rowgroup_max_buffer_mb); |
996 | | // Max buffer size for parquet chunk column |
997 | | DECLARE_mInt32(parquet_column_max_buffer_mb); |
998 | | // Merge small IO, the max amplified read ratio |
999 | | DECLARE_mDouble(max_amplified_read_ratio); |
1000 | | // Equivalent min size of each IO that can reach the maximum storage speed limit |
1001 | | // 1MB for oss, 8KB for hdfs |
1002 | | DECLARE_mInt32(merged_oss_min_io_size); |
1003 | | DECLARE_mInt32(merged_hdfs_min_io_size); |
1004 | | |
1005 | | // OrcReader |
1006 | | DECLARE_mInt32(orc_natural_read_size_mb); |
1007 | | DECLARE_mInt64(big_column_size_buffer); |
1008 | | DECLARE_mInt64(small_column_size_buffer); |
1009 | | |
1010 | | // When the rows number reached this limit, will check the filter rate the of bloomfilter |
1011 | | // if it is lower than a specific threshold, the predicate will be disabled. |
1012 | | DECLARE_mInt32(rf_predicate_check_row_num); |
1013 | | |
1014 | | // cooldown task configs |
1015 | | DECLARE_Int32(cooldown_thread_num); |
1016 | | DECLARE_mInt64(generate_cooldown_task_interval_sec); |
1017 | | DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h |
1018 | | DECLARE_mInt32(confirm_unused_remote_files_interval_sec); |
1019 | | DECLARE_Int32(cold_data_compaction_thread_num); |
1020 | | DECLARE_mInt32(cold_data_compaction_interval_sec); |
1021 | | |
1022 | | DECLARE_Int32(s3_transfer_executor_pool_size); |
1023 | | |
1024 | | DECLARE_Bool(enable_time_lut); |
1025 | | DECLARE_mBool(enable_simdjson_reader); |
1026 | | |
1027 | | DECLARE_mBool(enable_query_like_bloom_filter); |
1028 | | // number of s3 scanner thread pool size |
1029 | | DECLARE_Int32(doris_remote_scanner_thread_pool_thread_num); |
1030 | | // number of s3 scanner thread pool queue size |
1031 | | DECLARE_Int32(doris_remote_scanner_thread_pool_queue_size); |
1032 | | |
1033 | | // limit the queue of pending batches which will be sent by a single nodechannel |
1034 | | DECLARE_mInt64(nodechannel_pending_queue_max_bytes); |
1035 | | |
1036 | | // The batch size for sending data by brpc streaming client |
1037 | | DECLARE_mInt64(brpc_streaming_client_batch_bytes); |
1038 | | |
1039 | | DECLARE_Bool(enable_brpc_builtin_services); |
1040 | | |
1041 | | DECLARE_Bool(enable_brpc_connection_check); |
1042 | | |
1043 | | // Max waiting time to wait the "plan fragment start" rpc. |
1044 | | // If timeout, the fragment will be cancelled. |
1045 | | // This parameter is usually only used when the FE loses connection, |
1046 | | // and the BE can automatically cancel the relevant fragment after the timeout, |
1047 | | // so as to avoid occupying the execution thread for a long time. |
1048 | | DECLARE_mInt32(max_fragment_start_wait_time_seconds); |
1049 | | |
1050 | | DECLARE_Int32(fragment_mgr_cancel_worker_interval_seconds); |
1051 | | |
1052 | | // Node role tag for backend. Mix role is the default role, and computation role have no |
1053 | | // any tablet. |
1054 | | DECLARE_String(be_node_role); |
1055 | | |
1056 | | // Hide webserver page for safety. |
1057 | | // Hide the be config page for webserver. |
1058 | | DECLARE_Bool(hide_webserver_config_page); |
1059 | | |
1060 | | DECLARE_Bool(enable_segcompaction); |
1061 | | |
1062 | | // Max number of segments allowed in a single segcompaction task. |
1063 | | DECLARE_Int32(segcompaction_batch_size); |
1064 | | |
1065 | | // Max row count allowed in a single source segment, bigger segments will be skipped. |
1066 | | DECLARE_Int32(segcompaction_candidate_max_rows); |
1067 | | |
1068 | | // Max file size allowed in a single source segment, bigger segments will be skipped. |
1069 | | DECLARE_Int64(segcompaction_candidate_max_bytes); |
1070 | | |
1071 | | // Max total row count allowed in a single segcompaction task. |
1072 | | DECLARE_Int32(segcompaction_task_max_rows); |
1073 | | |
1074 | | // Max total file size allowed in a single segcompaction task. |
1075 | | DECLARE_Int64(segcompaction_task_max_bytes); |
1076 | | |
1077 | | // Global segcompaction thread pool size. |
1078 | | DECLARE_mInt32(segcompaction_num_threads); |
1079 | | |
1080 | | // enable java udf and jdbc scannode |
1081 | | DECLARE_Bool(enable_java_support); |
1082 | | |
1083 | | // Set config randomly to check more issues in github workflow |
1084 | | DECLARE_Bool(enable_fuzzy_mode); |
1085 | | |
1086 | | DECLARE_Bool(enable_debug_points); |
1087 | | |
1088 | | DECLARE_Int32(pipeline_executor_size); |
1089 | | |
1090 | | // block file cache |
1091 | | DECLARE_Bool(enable_file_cache); |
1092 | | // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}] |
1093 | | // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}] |
1094 | | // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,"normal_percent":85, "disposable_percent":10, "index_percent":5}] |
1095 | | DECLARE_String(file_cache_path); |
1096 | | DECLARE_Int64(async_file_cache_init_file_num_interval); |
1097 | | DECLARE_Int64(async_file_cache_init_sleep_interval_ms); |
1098 | | DECLARE_Int64(file_cache_min_file_segment_size); |
1099 | | DECLARE_Int64(file_cache_max_file_segment_size); |
1100 | | DECLARE_Bool(clear_file_cache); |
1101 | | DECLARE_Bool(enable_file_cache_query_limit); |
1102 | | // only for debug, will be removed after finding out the root cause |
1103 | | DECLARE_mInt32(file_cache_wait_sec_after_fail); // zero for no waiting and retrying |
1104 | | DECLARE_mInt32(file_cache_max_evict_num_per_round); |
1105 | | |
1106 | | // inverted index searcher cache |
1107 | | // cache entry stay time after lookup |
1108 | | DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s); |
1109 | | // cache entry that have not been visited for a certain period of time can be cleaned up by GC thread |
1110 | | DECLARE_mInt32(inverted_index_cache_stale_sweep_time_sec); |
1111 | | // inverted index searcher cache size |
1112 | | DECLARE_String(inverted_index_searcher_cache_limit); |
1113 | | // set `true` to enable insert searcher into cache when write inverted index data |
1114 | | DECLARE_Bool(enable_write_index_searcher_cache); |
1115 | | DECLARE_Bool(enable_inverted_index_cache_check_timestamp); |
1116 | | DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50% |
1117 | | DECLARE_Int32(inverted_index_query_cache_shards); |
1118 | | |
1119 | | // inverted index match bitmap cache size |
1120 | | DECLARE_String(inverted_index_query_cache_limit); |
1121 | | |
1122 | | // inverted index |
1123 | | DECLARE_mDouble(inverted_index_ram_buffer_size); |
1124 | | DECLARE_mInt32(inverted_index_max_buffered_docs); |
1125 | | // dict path for chinese analyzer |
1126 | | DECLARE_String(inverted_index_dict_path); |
1127 | | DECLARE_Int32(inverted_index_read_buffer_size); |
1128 | | // tree depth for bkd index |
1129 | | DECLARE_Int32(max_depth_in_bkd_tree); |
1130 | | // index compaction |
1131 | | DECLARE_mBool(inverted_index_compaction_enable); |
1132 | | // Only for debug, do not use in production |
1133 | | DECLARE_mBool(debug_inverted_index_compaction); |
1134 | | // index by RAM directory |
1135 | | DECLARE_mBool(inverted_index_ram_dir_enable); |
1136 | | // use num_broadcast_buffer blocks as buffer to do broadcast |
1137 | | DECLARE_Int32(num_broadcast_buffer); |
1138 | | |
1139 | | // max depth of expression tree allowed. |
1140 | | DECLARE_Int32(max_depth_of_expr_tree); |
1141 | | |
1142 | | // Report a tablet as bad when io errors occurs more than this value. |
1143 | | DECLARE_mInt64(max_tablet_io_errors); |
1144 | | |
1145 | | // Report a tablet as bad when its path not found |
1146 | | DECLARE_Int32(tablet_path_check_interval_seconds); |
1147 | | DECLARE_mInt32(tablet_path_check_batch_size); |
1148 | | |
1149 | | // it must be larger than or equal to 5MB |
1150 | | DECLARE_mInt32(s3_write_buffer_size); |
1151 | | // The timeout config for S3 buffer allocation |
1152 | | DECLARE_mInt32(s3_writer_buffer_allocation_timeout); |
1153 | | // the max number of cached file handle for block segemnt |
1154 | | DECLARE_mInt64(file_cache_max_file_reader_cache_size); |
1155 | | //enable shrink memory |
1156 | | DECLARE_mBool(enable_shrink_memory); |
1157 | | // enable cache for high concurrent point query work load |
1158 | | DECLARE_mInt32(schema_cache_capacity); |
1159 | | DECLARE_mInt32(schema_cache_sweep_time_sec); |
1160 | | |
1161 | | // max number of segment cache |
1162 | | DECLARE_Int32(segment_cache_capacity); |
1163 | | DECLARE_Int32(segment_cache_fd_percentage); |
1164 | | DECLARE_Int32(segment_cache_memory_percentage); |
1165 | | DECLARE_mInt32(estimated_mem_per_column_reader); |
1166 | | |
1167 | | // enable binlog |
1168 | | DECLARE_Bool(enable_feature_binlog); |
1169 | | |
1170 | | // enable set in BitmapValue |
1171 | | DECLARE_Bool(enable_set_in_bitmap_value); |
1172 | | |
1173 | | // max number of hdfs file handle in cache |
1174 | | DECLARE_Int64(max_hdfs_file_handle_cache_num); |
1175 | | DECLARE_Int32(max_hdfs_file_handle_cache_time_sec); |
1176 | | |
1177 | | // max number of meta info of external files, such as parquet footer |
1178 | | DECLARE_Int64(max_external_file_meta_cache_num); |
1179 | | // Apply delete pred in cumu compaction |
1180 | | DECLARE_mBool(enable_delete_when_cumu_compaction); |
1181 | | |
1182 | | // max_write_buffer_number for rocksdb |
1183 | | DECLARE_Int32(rocksdb_max_write_buffer_number); |
1184 | | |
1185 | | // Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false. |
1186 | | DECLARE_mBool(allow_zero_date); |
1187 | | // Allow invalid decimalv2 literal for compatible with old version. Recommend set it false strongly. |
1188 | | DECLARE_mBool(allow_invalid_decimalv2_literal); |
1189 | | // Allow to specify kerberos credentials cache path. |
1190 | | DECLARE_mString(kerberos_ccache_path); |
1191 | | // set krb5.conf path, use "/etc/krb5.conf" by default |
1192 | | DECLARE_mString(kerberos_krb5_conf_path); |
1193 | | |
1194 | | // Values include `none`, `glog`, `boost`, `glibc`, `libunwind` |
1195 | | DECLARE_mString(get_stack_trace_tool); |
1196 | | DECLARE_mBool(enable_address_sanitizers_with_stack_trace); |
1197 | | |
1198 | | // DISABLED: Don't resolve location info. |
1199 | | // FAST: Perform CU lookup using .debug_aranges (might be incomplete). |
1200 | | // FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure. |
1201 | | // FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL. |
1202 | | DECLARE_mString(dwarf_location_info_mode); |
1203 | | |
1204 | | // the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer |
1205 | | DECLARE_mInt64(auto_inc_prefetch_size_ratio); |
1206 | | |
1207 | | // the ratio of _low_level_water_level_mark/_batch_size in AutoIncIDBuffer |
1208 | | DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio); |
1209 | | |
1210 | | // number of threads that fetch auto-inc ranges from FE |
1211 | | DECLARE_mInt64(auto_inc_fetch_thread_num); |
1212 | | // Max connection cache num for point lookup queries |
1213 | | DECLARE_mInt64(lookup_connection_cache_bytes_limit); |
1214 | | |
1215 | | // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT |
1216 | | DECLARE_mInt64(LZ4_HC_compression_level); |
1217 | | // Whether flatten nested arrays in variant column |
1218 | | // Notice: TEST ONLY |
1219 | | DECLARE_mBool(variant_enable_flatten_nested); |
1220 | | // Threshold of a column as sparse column |
1221 | | // Notice: TEST ONLY |
1222 | | DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column); |
1223 | | // Threshold to estimate a column is sparsed |
1224 | | // Notice: TEST ONLY |
1225 | | DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column); |
1226 | | // Treat invalid json format str as string, instead of throwing exception if false |
1227 | | DECLARE_mBool(variant_throw_exeception_on_invalid_json); |
1228 | | |
1229 | | DECLARE_mBool(enable_merge_on_write_correctness_check); |
1230 | | // rowid conversion correctness check when compaction for mow table |
1231 | | DECLARE_mBool(enable_rowid_conversion_correctness_check); |
1232 | | // missing rows correctness check when compaction for mow table |
1233 | | DECLARE_mBool(enable_missing_rows_correctness_check); |
1234 | | // When the number of missing versions is more than this value, do not directly |
1235 | | // retry the publish and handle it through async publish. |
1236 | | DECLARE_mInt32(mow_publish_max_discontinuous_version_num); |
1237 | | // When the version is not continuous for MOW table in publish phase and the gap between |
1238 | | // current txn's publishing version and the max version of the tablet exceeds this value, |
1239 | | // don't print warning log |
1240 | | DECLARE_mInt32(publish_version_gap_logging_threshold); |
1241 | | |
1242 | | // The secure path with user files, used in the `local` table function. |
1243 | | DECLARE_mString(user_files_secure_path); |
1244 | | |
1245 | | // If fe's frontend info has not been updated for more than fe_expire_duration_seconds, it will be regarded |
1246 | | // as an abnormal fe, this will cause be to cancel this fe's related query. |
1247 | | DECLARE_Int32(fe_expire_duration_seconds); |
1248 | | |
1249 | | // If use stop_be.sh --grace, then BE has to wait all running queries to stop to avoiding running query failure |
1250 | | // , but if the waiting time exceed the limit, then be will exit directly. |
1251 | | // During this period, FE will not send any queries to BE and waiting for all running queries to stop. |
1252 | | DECLARE_Int32(grace_shutdown_wait_seconds); |
1253 | | |
1254 | | // BitmapValue serialize version. |
1255 | | DECLARE_Int16(bitmap_serialize_version); |
1256 | | |
1257 | | // group commit config |
1258 | | DECLARE_String(group_commit_wal_path); |
1259 | | DECLARE_Int32(group_commit_replay_wal_retry_num); |
1260 | | DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds); |
1261 | | DECLARE_Int32(group_commit_replay_wal_retry_interval_max_seconds); |
1262 | | DECLARE_mInt32(group_commit_relay_wal_threads); |
1263 | | // This config can be set to limit thread number in group commit request fragment thread pool. |
1264 | | DECLARE_mInt32(group_commit_insert_threads); |
1265 | | DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio); |
1266 | | DECLARE_Bool(wait_internal_group_commit_finish); |
1267 | | // Max size(bytes) of group commit queues, used for mem back pressure. |
1268 | | DECLARE_mInt32(group_commit_queue_mem_limit); |
1269 | | // Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space. |
1270 | | // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified. |
1271 | | DECLARE_mString(group_commit_wal_max_disk_limit); |
1272 | | DECLARE_Bool(group_commit_wait_replay_wal_finish); |
1273 | | |
1274 | | // The configuration item is used to lower the priority of the scanner thread, |
1275 | | // typically employed to ensure CPU scheduling for write operations. |
1276 | | // Default is 0, which is default value of thread nice value, increase this value |
1277 | | // to lower the priority of scan threads |
1278 | | DECLARE_Int32(scan_thread_nice_value); |
1279 | | // Used to modify the recycle interval of tablet schema cache |
1280 | | DECLARE_mInt32(tablet_schema_cache_recycle_interval); |
1281 | | // Granularity is at the column level |
1282 | | DECLARE_mInt32(tablet_schema_cache_capacity); |
1283 | | |
1284 | | // Use `LOG(FATAL)` to replace `throw` when true |
1285 | | DECLARE_mBool(exit_on_exception); |
1286 | | |
1287 | | // cgroup |
1288 | | DECLARE_mString(doris_cgroup_cpu_path); |
1289 | | |
1290 | | DECLARE_mBool(enable_workload_group_memory_gc); |
1291 | | |
1292 | | // This config controls whether the s3 file writer would flush cache asynchronously |
1293 | | DECLARE_Bool(enable_flush_file_cache_async); |
1294 | | |
1295 | | // Remove predicate that is always true for a segment. |
1296 | | DECLARE_Bool(ignore_always_true_predicate_for_segment); |
1297 | | |
1298 | | // Ingest binlog work pool size |
1299 | | DECLARE_Int32(ingest_binlog_work_pool_size); |
1300 | | |
1301 | | // Download binlog rate limit, unit is KB/s |
1302 | | DECLARE_Int32(download_binlog_rate_limit_kbs); |
1303 | | |
1304 | | DECLARE_mInt32(buffered_reader_read_timeout_ms); |
1305 | | |
1306 | | // whether to enable /api/snapshot api |
1307 | | DECLARE_Bool(enable_snapshot_action); |
1308 | | |
1309 | | // The max columns size for a tablet schema |
1310 | | DECLARE_mInt32(variant_max_merged_tablet_schema_size); |
1311 | | |
1312 | | DECLARE_mInt64(local_exchange_buffer_mem_limit); |
1313 | | |
1314 | | DECLARE_mInt64(enable_debug_log_timeout_secs); |
1315 | | |
1316 | | DECLARE_mBool(enable_column_type_check); |
1317 | | |
1318 | | // Tolerance for the number of partition id 0 in rowset, default 0 |
1319 | | DECLARE_Int32(ignore_invalid_partition_id_rowset_num); |
1320 | | |
1321 | | DECLARE_mInt32(report_query_statistics_interval_ms); |
1322 | | DECLARE_mInt32(query_statistics_reserve_timeout_ms); |
1323 | | |
1324 | | // consider two high usage disk at the same available level if they do not exceed this diff. |
1325 | | DECLARE_mDouble(high_disk_avail_level_diff_usages); |
1326 | | |
1327 | | // create tablet in partition random robin idx lru size, default 10000 |
1328 | | DECLARE_Int32(partition_disk_index_lru_size); |
1329 | | DECLARE_String(spill_storage_root_path); |
1330 | | // Spill storage limit specified as number of bytes |
1331 | | // ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'), |
1332 | | // or percentage of capaity ('<int>%'). |
1333 | | // Defaults to bytes if no unit is given. |
1334 | | // Must larger than 0. |
1335 | | // If specified as percentage, the final limit value is: |
1336 | | // disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit |
1337 | | DECLARE_String(spill_storage_limit); |
1338 | | DECLARE_mInt32(spill_gc_interval_ms); |
1339 | | DECLARE_mInt32(spill_gc_work_time_ms); |
1340 | | DECLARE_Int32(spill_io_thread_pool_thread_num); |
1341 | | DECLARE_Int32(spill_io_thread_pool_queue_size); |
1342 | | |
1343 | | DECLARE_mBool(check_segment_when_build_rowset_meta); |
1344 | | |
1345 | | // max s3 client retry times |
1346 | | DECLARE_mInt32(max_s3_client_retry); |
1347 | | // When meet s3 429 error, the "get" request will |
1348 | | // sleep s3_read_base_wait_time_ms (*1, *2, *3, *4) ms |
1349 | | // get try again. |
1350 | | // The max sleep time is s3_read_max_wait_time_ms |
1351 | | // and the max retry time is max_s3_client_retry |
1352 | | DECLARE_mInt32(s3_read_base_wait_time_ms); |
1353 | | DECLARE_mInt32(s3_read_max_wait_time_ms); |
1354 | | |
1355 | | // write as inverted index tmp directory |
1356 | | DECLARE_String(tmp_file_dir); |
1357 | | |
1358 | | // the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client |
1359 | | DECLARE_mString(ca_cert_file_paths); |
1360 | | |
1361 | | /** Table sink configurations(currently contains only external table types) **/ |
1362 | | // Minimum data processed to scale writers in exchange when non partition writing |
1363 | | DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold); |
1364 | | // Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing |
1365 | | DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold); |
1366 | | // Minimum partition data processed to rebalance writers in exchange when partition writing |
1367 | | DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold); |
1368 | | // Maximum processed partition nums of per writer when partition writing |
1369 | | DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer); |
1370 | | |
1371 | | /** Hive sink configurations **/ |
1372 | | DECLARE_mInt64(hive_sink_max_file_size); |
1373 | | |
1374 | | /** Iceberg sink configurations **/ |
1375 | | DECLARE_mInt64(iceberg_sink_max_file_size); |
1376 | | |
1377 | | // Number of open tries, default 1 means only try to open once. |
1378 | | // Retry the Open num_retries time waiting 100 milliseconds between retries. |
1379 | | DECLARE_mInt32(thrift_client_open_num_tries); |
1380 | | |
1381 | | DECLARE_mBool(ignore_schema_change_check); |
1382 | | |
1383 | | //JVM monitoring enable. To prevent be from crashing due to jvm compatibility issues. |
1384 | | DECLARE_Bool(enable_jvm_monitor); |
1385 | | |
1386 | | // Skip loading stale rowset meta when initializing `TabletMeta` from protobuf |
1387 | | DECLARE_mBool(skip_loading_stale_rowset_meta); |
1388 | | // Whether to use file to record log. When starting BE with --console, |
1389 | | // all logs will be written to both standard output and file. |
1390 | | // Disable this option will no longer use file to record log. |
1391 | | // Only works when starting BE with --console. |
1392 | | DECLARE_Bool(enable_file_logger); |
1393 | | // The time out milliseconds for remote fetch schema RPC |
1394 | | DECLARE_mInt64(fetch_remote_schema_rpc_timeout_ms); |
1395 | | |
1396 | | // The minimum row group size when exporting Parquet files. |
1397 | | DECLARE_Int64(min_row_group_size); |
1398 | | |
1399 | | DECLARE_mInt64(compaction_memory_bytes_limit); |
1400 | | |
1401 | | DECLARE_mInt64(compaction_batch_size); |
1402 | | |
1403 | | DECLARE_mBool(enable_parquet_page_index); |
1404 | | |
1405 | | // Wheather to ignore not found file in external teble(eg, hive) |
1406 | | // Default is true, if set to false, the not found file will result in query failure. |
1407 | | DECLARE_mBool(ignore_not_found_file_in_external_table); |
1408 | | |
1409 | | DECLARE_mInt64(tablet_meta_serialize_size_limit); |
1410 | | |
1411 | | DECLARE_mInt64(pipeline_task_leakage_detect_period_secs); |
1412 | | // To be compatible with hadoop's block compression |
1413 | | DECLARE_mInt32(snappy_compression_block_size); |
1414 | | DECLARE_mInt32(lz4_compression_block_size); |
1415 | | |
1416 | | DECLARE_mBool(enable_pipeline_task_leakage_detect); |
1417 | | |
1418 | | #ifdef BE_TEST |
1419 | | // test s3 |
1420 | | DECLARE_String(test_s3_resource); |
1421 | | DECLARE_String(test_s3_ak); |
1422 | | DECLARE_String(test_s3_sk); |
1423 | | DECLARE_String(test_s3_endpoint); |
1424 | | DECLARE_String(test_s3_region); |
1425 | | DECLARE_String(test_s3_bucket); |
1426 | | DECLARE_String(test_s3_prefix); |
1427 | | #endif |
1428 | | |
1429 | | class Register { |
1430 | | public: |
1431 | | struct Field { |
1432 | | const char* type = nullptr; |
1433 | | const char* name = nullptr; |
1434 | | void* storage = nullptr; |
1435 | | const char* defval = nullptr; |
1436 | | bool valmutable = false; |
1437 | | Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval, |
1438 | | bool fvalmutable) |
1439 | | : type(ftype), |
1440 | | name(fname), |
1441 | | storage(fstorage), |
1442 | | defval(fdefval), |
1443 | 544 | valmutable(fvalmutable) {} |
1444 | | }; |
1445 | | |
1446 | | public: |
1447 | | static std::map<std::string, Field>* _s_field_map; |
1448 | | |
1449 | | public: |
1450 | | Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval, |
1451 | 540 | bool fvalmutable) { |
1452 | 540 | if (_s_field_map == nullptr) { |
1453 | 1 | _s_field_map = new std::map<std::string, Field>(); |
1454 | 1 | } |
1455 | 540 | Field field(ftype, fname, fstorage, fdefval, fvalmutable); |
1456 | 540 | _s_field_map->insert(std::make_pair(std::string(fname), field)); |
1457 | 540 | } |
1458 | | }; |
1459 | | |
1460 | | // RegisterConfValidator class is used to store validator function of registered config fields in |
1461 | | // Register::_s_field_map. |
1462 | | // If any validator return false when BE bootstart, the bootstart will be terminated. |
1463 | | // If validator return false when use http API to update some config, the config will not |
1464 | | // be modified and the API will return failure. |
1465 | | class RegisterConfValidator { |
1466 | | public: |
1467 | | // Validator for each config name. |
1468 | | static std::map<std::string, std::function<bool()>>* _s_field_validator; |
1469 | | |
1470 | | public: |
1471 | | RegisterConfValidator(const char* fname, const std::function<bool()>& validator) { |
1472 | | if (_s_field_validator == nullptr) { |
1473 | | _s_field_validator = new std::map<std::string, std::function<bool()>>(); |
1474 | | } |
1475 | | // register validator to _s_field_validator |
1476 | | _s_field_validator->insert(std::make_pair(std::string(fname), validator)); |
1477 | | } |
1478 | | }; |
1479 | | |
1480 | | // configuration properties load from config file. |
1481 | | class Properties { |
1482 | | public: |
1483 | | // load conf from file, if must_exist is true and file does not exist, return false |
1484 | | bool load(const char* conf_file, bool must_exist = true); |
1485 | | |
1486 | | // Find the config value by key from `file_conf_map`. |
1487 | | // If found, set `retval` to the config value, |
1488 | | // or set `retval` to `defstr` |
1489 | | // if retval is not set(in case defstr is nullptr), set is_retval_set to false |
1490 | | template <typename T> |
1491 | | bool get_or_default(const char* key, const char* defstr, T& retval, bool* is_retval_set, |
1492 | | std::string& rawval) const; |
1493 | | |
1494 | | void set(const std::string& key, const std::string& val); |
1495 | | |
1496 | | void set_force(const std::string& key, const std::string& val); |
1497 | | |
1498 | | // dump props to conf file |
1499 | | Status dump(const std::string& conffile); |
1500 | | |
1501 | | private: |
1502 | | std::map<std::string, std::string> file_conf_map; |
1503 | | }; |
1504 | | |
1505 | | // full configurations. |
1506 | | extern std::map<std::string, std::string>* full_conf_map; |
1507 | | |
1508 | | extern std::mutex custom_conf_lock; |
1509 | | |
1510 | | // Init the config from `conf_file`. |
1511 | | // If fill_conf_map is true, the updated config will also update the `full_conf_map`. |
1512 | | // If must_exist is true and `conf_file` does not exist, this function will return false. |
1513 | | // If set_to_default is true, the config value will be set to default value if not found in `conf_file`. |
1514 | | bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist = true, |
1515 | | bool set_to_default = true); |
1516 | | |
1517 | | Status set_config(const std::string& field, const std::string& value, bool need_persist = false, |
1518 | | bool force = false); |
1519 | | |
1520 | | Status persist_config(const std::string& field, const std::string& value); |
1521 | | |
1522 | | std::mutex* get_mutable_string_config_lock(); |
1523 | | |
1524 | | std::vector<std::vector<std::string>> get_config_info(); |
1525 | | |
1526 | | Status set_fuzzy_configs(); |
1527 | | |
1528 | | void update_config(const std::string& field, const std::string& value); |
1529 | | |
1530 | | } // namespace config |
1531 | | } // namespace doris |