/root/doris/be/src/common/logconfig.cpp

Source
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <ctype.h>
#include <stdint.h>

#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <mutex>
#include <string>
#include <vector>

#include "common/config.h"
#include "common/logging.h"

namespace doris {
#include "common/compile_check_avoid_begin.h"

static bool logging_initialized = false;

static std::mutex logging_mutex;

// Implement the custom log format: I20250118 10:53:06.239614 1318521 timezone_utils.cpp:115] Preloaded653 timezones.
struct StdoutLogSink : google::LogSink {
    void send(google::LogSeverity severity, const char* /*full_filename*/,
              const char* base_filename, int line, const google::LogMessageTime& time,
              const char* message, std::size_t message_len) override {
        // 1.  Convert log severity to corresponding character (I/W/E/F)
        char severity_char;
        switch (severity) {
        case google::GLOG_INFO:
            severity_char = 'I';
            break;
        case google::GLOG_WARNING:
            severity_char = 'W';
            break;
        case google::GLOG_ERROR:
            severity_char = 'E';
            break;
        case google::GLOG_FATAL:
            severity_char = 'F';
            break;
        default:
            severity_char = '?';
            break;
        }
        // Set output formatting flags
        std::cout << std::setfill('0');

        // 1. Log severity (I/W/E/F)
        std::cout << severity_char;

        // 2. Date (YYYYMMDD)
        // Note: tm_year is years since 1900, tm_mon is 0-based (0-11)
        std::cout << std::setw(4) << (time.year() + 1900) << std::setw(2) << std::setfill('0')
                  << (time.month() + 1) << std::setw(2) << std::setfill('0') << time.day();

        // 3. Time (HH:MM:SS.ffffff)
        std::cout << " " << std::setw(2) << std::setfill('0') << time.hour() << ":" << std::setw(2)
                  << std::setfill('0') << time.min() << ":" << std::setw(2) << std::setfill('0')
                  << time.sec() << "." << std::setw(6) << std::setfill('0') << time.usec();

        // 4. Process ID
        std::cout << " " << getpid();

        // 5. Filename and line number
        std::cout << " " << base_filename << ":" << line << "] ";

        // 6. Log message
        std::cout.write(message, message_len);

        // Add newline and flush
        std::cout << std::endl;
    }
};

static StdoutLogSink stdout_log_sink;

static bool iequals(const std::string& a, const std::string& b) {
    unsigned int sz = a.size();
    if (b.size() != sz) {
        return false;
    }
    for (unsigned int i = 0; i < sz; ++i) {
        if (tolower(a[i]) != tolower(b[i])) {
            return false;
        }
    }
    return true;
}

// if custom_date_time_format = false, same format as in be.log
// The following is same as default log format. eg:
// I20240605 15:25:15.677153 1763151 wal_manager.cpp:481] msg...
template <bool add_runtime_logger_prefix = false, bool custom_date_time_format = false>
void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void* arg) {
    if constexpr (add_runtime_logger_prefix) {
        // Add prefix "RuntimeLogger ".
        s << "RuntimeLogger ";
    }
    s << l.severity[0];

    // Add a space if custom_date_time_format.
    if constexpr (custom_date_time_format) {
        s << ' ';
    }

    std::tm tm_time = {};
    tm_time.tm_year = l.time.year();
    tm_time.tm_mon = l.time.month();
    tm_time.tm_mday = l.time.day();
    tm_time.tm_hour = l.time.hour();
    tm_time.tm_min = l.time.min();
    tm_time.tm_sec = l.time.sec();

    if constexpr (custom_date_time_format) {
        s << std::put_time(&tm_time, config::sys_log_custom_date_time_format.c_str());
        if (!config::sys_log_custom_date_time_ms_format.empty()) {
            s << fmt::format(config::sys_log_custom_date_time_ms_format, l.time.usec() / 1000);
        }
    } else {
        s << std::put_time(&tm_time, "%Y%m%d %H:%M:%S");
        s << "." << std::setw(6) << l.time.usec();
    }

    s << ' ';
    s << std::setfill(' ') << std::setw(5);
    s << l.thread_id << std::setfill('0');
    s << ' ';
    s << l.filename << ':' << l.line_number << "]";
}

bool init_glog(const char* basename) {
    std::lock_guard<std::mutex> logging_lock(logging_mutex);

    if (logging_initialized) {
        return true;
    }

    bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
    if (log_to_console) {
        if (doris::config::enable_file_logger) {
            // will output log to be.info and output log to stdout
            google::AddLogSink(&stdout_log_sink);
        } else {
            // enable_file_logger is false, will only output log to stdout
            // Not output to stderr because be.out will output log to stderr
            FLAGS_logtostdout = true;
        }
    }

    // don't log to stderr except fatal level
    // so fatal log can output to be.out .
    FLAGS_stderrthreshold = google::FATAL;
    // set glog log dir
    // ATTN: sys_log_dir is deprecated, this is just for compatibility
    std::string log_dir = config::sys_log_dir;
    if (log_dir == "") {
        log_dir = getenv("LOG_DIR");
    }
    FLAGS_log_dir = log_dir;
    // 0 means buffer INFO only
    FLAGS_logbuflevel = 0;
    // buffer log messages for at most this many seconds
    FLAGS_logbufsecs = 30;
    // set roll num
    FLAGS_log_filenum_quota = config::sys_log_roll_num;

    // set log level
    std::string& loglevel = config::sys_log_level;
    if (iequals(loglevel, "INFO")) {
        FLAGS_minloglevel = 0;
    } else if (iequals(loglevel, "WARNING")) {
        FLAGS_minloglevel = 1;
    } else if (iequals(loglevel, "ERROR")) {
        FLAGS_minloglevel = 2;
    } else if (iequals(loglevel, "FATAL")) {
        FLAGS_minloglevel = 3;
    } else {
        std::cerr << "sys_log_level needs to be INFO, WARNING, ERROR, FATAL" << std::endl;
        return false;
    }

    // set log buffer level
    // default is 0
    std::string& logbuflevel = config::log_buffer_level;
    if (iequals(logbuflevel, "-1")) {
        FLAGS_logbuflevel = -1;
    } else if (iequals(logbuflevel, "0")) {
        FLAGS_logbuflevel = 0;
    }

    // set log roll mode
    std::string& rollmode = config::sys_log_roll_mode;
    std::string sizeflag = "SIZE-MB-";
    bool ok = false;
    if (rollmode.compare("TIME-DAY") == 0) {
        FLAGS_log_split_method = "day";
        ok = true;
    } else if (rollmode.compare("TIME-HOUR") == 0) {
        FLAGS_log_split_method = "hour";
        ok = true;
    } else if (rollmode.substr(0, sizeflag.length()).compare(sizeflag) == 0) {
        FLAGS_log_split_method = "size";
        std::string sizestr = rollmode.substr(sizeflag.size(), rollmode.size() - sizeflag.size());
        if (sizestr.size() != 0) {
            char* end = nullptr;
            errno = 0;
            const char* sizecstr = sizestr.c_str();
            int64_t ret64 = strtoll(sizecstr, &end, 10);
            if ((errno == 0) && (end == sizecstr + strlen(sizecstr))) {
                int32_t retval = static_cast<int32_t>(ret64);
                if (retval == ret64) {
                    FLAGS_max_log_size = retval;
                    ok = true;
                }
            }
        }
    } else {
        ok = false;
    }
    if (!ok) {
        std::cerr << "sys_log_roll_mode needs to be TIME-DAY, TIME-HOUR, SIZE-MB-nnn" << std::endl;
        return false;
    }

    // set verbose modules.
    FLAGS_v = config::sys_log_verbose_flags_v;
    std::vector<std::string>& verbose_modules = config::sys_log_verbose_modules;
    int32_t vlog_level = config::sys_log_verbose_level;
    for (size_t i = 0; i < verbose_modules.size(); i++) {
        if (verbose_modules[i].size() != 0) {
            google::SetVLOGLevel(verbose_modules[i].c_str(), vlog_level);
        }
    }

    if (log_to_console) {
        // Add prefix if log output to stderr
        if (config::sys_log_enable_custom_date_time_format) {
            google::InitGoogleLogging(basename, &custom_prefix<true, true>);
        } else {
            google::InitGoogleLogging(basename, &custom_prefix<true, false>);
        }
    } else {
        if (config::sys_log_enable_custom_date_time_format) {
            google::InitGoogleLogging(basename, &custom_prefix<false, true>);
        } else {
            google::InitGoogleLogging(basename);
        }
    }

    logging_initialized = true;

    return true;
}

void shutdown_logging() {
    std::lock_guard<std::mutex> logging_lock(logging_mutex);
    google::RemoveLogSink(&stdout_log_sink);
    google::ShutdownGoogleLogging();
}

void update_logging(const std::string& name, const std::string& value) {
    if ("sys_log_level" == name) {
        if (iequals(value, "INFO")) {
            FLAGS_minloglevel = 0;
        } else if (iequals(value, "WARNING")) {
            FLAGS_minloglevel = 1;
        } else if (iequals(value, "ERROR")) {
            FLAGS_minloglevel = 2;
        } else if (iequals(value, "FATAL")) {
            FLAGS_minloglevel = 3;
        } else {
            LOG(WARNING) << "update sys_log_level failed, need to be INFO, WARNING, ERROR, FATAL";
        }
    }
}

#include "common/compile_check_avoid_end.h"
} // namespace doris

Coverage Report

Created: 2025-09-12 10:50

Line	Count	Source
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17
18		#include <ctype.h>
19		#include <stdint.h>
20
21		#include <cerrno>
22		#include <cstdlib>
23		#include <cstring>
24		#include <iomanip>
25		#include <iostream>
26		#include <mutex>
27		#include <string>
28		#include <vector>
29
30		#include "common/config.h"
31		#include "common/logging.h"
32
33		namespace doris {
34		#include "common/compile_check_avoid_begin.h"
35
36		static bool logging_initialized = false;
37
38		static std::mutex logging_mutex;
39
40		// Implement the custom log format: I20250118 10:53:06.239614 1318521 timezone_utils.cpp:115] Preloaded653 timezones.
41		struct StdoutLogSink : google::LogSink {
42		void send(google::LogSeverity severity, const char* /full_filename/,
43		const char* base_filename, int line, const google::LogMessageTime& time,
44	0	const char* message, std::size_t message_len) override {
45		// 1. Convert log severity to corresponding character (I/W/E/F)
46	0	char severity_char;
47	0	switch (severity) {
48	0	case google::GLOG_INFO:
49	0	severity_char = 'I';
50	0	break;
51	0	case google::GLOG_WARNING:
52	0	severity_char = 'W';
53	0	break;
54	0	case google::GLOG_ERROR:
55	0	severity_char = 'E';
56	0	break;
57	0	case google::GLOG_FATAL:
58	0	severity_char = 'F';
59	0	break;
60	0	default:
61	0	severity_char = '?';
62	0	break;
63	0	}
64		// Set output formatting flags
65	0	std::cout << std::setfill('0');
66
67		// 1. Log severity (I/W/E/F)
68	0	std::cout << severity_char;
69
70		// 2. Date (YYYYMMDD)
71		// Note: tm_year is years since 1900, tm_mon is 0-based (0-11)
72	0	std::cout << std::setw(4) << (time.year() + 1900) << std::setw(2) << std::setfill('0')
73	0	<< (time.month() + 1) << std::setw(2) << std::setfill('0') << time.day();
74
75		// 3. Time (HH:MM:SS.ffffff)
76	0	std::cout << " " << std::setw(2) << std::setfill('0') << time.hour() << ":" << std::setw(2)
77	0	<< std::setfill('0') << time.min() << ":" << std::setw(2) << std::setfill('0')
78	0	<< time.sec() << "." << std::setw(6) << std::setfill('0') << time.usec();
79
80		// 4. Process ID
81	0	std::cout << " " << getpid();
82
83		// 5. Filename and line number
84	0	std::cout << " " << base_filename << ":" << line << "] ";
85
86		// 6. Log message
87	0	std::cout.write(message, message_len);
88
89		// Add newline and flush
90	0	std::cout << std::endl;
91	0	}
92		};
93
94		static StdoutLogSink stdout_log_sink;
95
96	15	static bool iequals(const std::string& a, const std::string& b) {
97	15	unsigned int sz = a.size();
98	15	if (b.size() != sz) {
99	2	return false;
100	2	}
101	53	for (unsigned int i = 0; i < sz; ++i) {
102	40	if (tolower(a[i]) != tolower(b[i])) {
103	0	return false;
104	0	}
105	40	}
106	13	return true;
107	13	}
108
109		// if custom_date_time_format = false, same format as in be.log
110		// The following is same as default log format. eg:
111		// I20240605 15:25:15.677153 1763151 wal_manager.cpp:481] msg...
112		template <bool add_runtime_logger_prefix = false, bool custom_date_time_format = false>
113	0	void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void* arg) {
114	0	if constexpr (add_runtime_logger_prefix) {
115		// Add prefix "RuntimeLogger ".
116	0	s << "RuntimeLogger ";
117	0	}
118	0	s << l.severity[0];
119
120		// Add a space if custom_date_time_format.
121	0	if constexpr (custom_date_time_format) {
122	0	s << ' ';
123	0	}
124
125	0	std::tm tm_time = {};
126	0	tm_time.tm_year = l.time.year();
127	0	tm_time.tm_mon = l.time.month();
128	0	tm_time.tm_mday = l.time.day();
129	0	tm_time.tm_hour = l.time.hour();
130	0	tm_time.tm_min = l.time.min();
131	0	tm_time.tm_sec = l.time.sec();
132
133	0	if constexpr (custom_date_time_format) {
134	0	s << std::put_time(&tm_time, config::sys_log_custom_date_time_format.c_str());
135	0	if (!config::sys_log_custom_date_time_ms_format.empty()) {
136	0	s << fmt::format(config::sys_log_custom_date_time_ms_format, l.time.usec() / 1000);
137	0	}
138	0	} else {
139	0	s << std::put_time(&tm_time, "%Y%m%d %H:%M:%S");
140	0	s << "." << std::setw(6) << l.time.usec();
141	0	}
142
143	0	s << ' ';
144	0	s << std::setfill(' ') << std::setw(5);
145	0	s << l.thread_id << std::setfill('0');
146	0	s << ' ';
147	0	s << l.filename << ':' << l.line_number << "]";
148	0	} Unexecuted instantiation: _ZN5doris13custom_prefixILb1ELb1EEEvRSoRKN6google14LogMessageInfoEPv Unexecuted instantiation: _ZN5doris13custom_prefixILb1ELb0EEEvRSoRKN6google14LogMessageInfoEPv Unexecuted instantiation: _ZN5doris13custom_prefixILb0ELb1EEEvRSoRKN6google14LogMessageInfoEPv
149
150	7	bool init_glog(const char* basename) {
151	7	std::lock_guard<std::mutex> logging_lock(logging_mutex);
152
153	7	if (logging_initialized) {
154	0	return true;
155	0	}
156
157	7	bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
158	7	if (log_to_console) {
159	0	if (doris::config::enable_file_logger) {
160		// will output log to be.info and output log to stdout
161	0	google::AddLogSink(&stdout_log_sink);
162	0	} else {
163		// enable_file_logger is false, will only output log to stdout
164		// Not output to stderr because be.out will output log to stderr
165	0	FLAGS_logtostdout = true;
166	0	}
167	0	}
168
169		// don't log to stderr except fatal level
170		// so fatal log can output to be.out .
171	7	FLAGS_stderrthreshold = google::FATAL;
172		// set glog log dir
173		// ATTN: sys_log_dir is deprecated, this is just for compatibility
174	7	std::string log_dir = config::sys_log_dir;
175	7	if (log_dir == "") {
176	7	log_dir = getenv("LOG_DIR");
177	7	}
178	7	FLAGS_log_dir = log_dir;
179		// 0 means buffer INFO only
180	7	FLAGS_logbuflevel = 0;
181		// buffer log messages for at most this many seconds
182	7	FLAGS_logbufsecs = 30;
183		// set roll num
184	7	FLAGS_log_filenum_quota = config::sys_log_roll_num;
185
186		// set log level
187	7	std::string& loglevel = config::sys_log_level;
188	7	if (iequals(loglevel, "INFO")) {
189	7	FLAGS_minloglevel = 0;
190	7	} else if (iequals(loglevel, "WARNING")) {
191	0	FLAGS_minloglevel = 1;
192	0	} else if (iequals(loglevel, "ERROR")) {
193	0	FLAGS_minloglevel = 2;
194	0	} else if (iequals(loglevel, "FATAL")) {
195	0	FLAGS_minloglevel = 3;
196	0	} else {
197	0	std::cerr << "sys_log_level needs to be INFO, WARNING, ERROR, FATAL" << std::endl;
198	0	return false;
199	0	}
200
201		// set log buffer level
202		// default is 0
203	7	std::string& logbuflevel = config::log_buffer_level;
204	7	if (iequals(logbuflevel, "-1")) {
205	6	FLAGS_logbuflevel = -1;
206	6	} else if (iequals(logbuflevel, "0")) {
207	0	FLAGS_logbuflevel = 0;
208	0	}
209
210		// set log roll mode
211	7	std::string& rollmode = config::sys_log_roll_mode;
212	7	std::string sizeflag = "SIZE-MB-";
213	7	bool ok = false;
214	7	if (rollmode.compare("TIME-DAY") == 0) {
215	0	FLAGS_log_split_method = "day";
216	0	ok = true;
217	7	} else if (rollmode.compare("TIME-HOUR") == 0) {
218	0	FLAGS_log_split_method = "hour";
219	0	ok = true;
220	7	} else if (rollmode.substr(0, sizeflag.length()).compare(sizeflag) == 0) {
221	7	FLAGS_log_split_method = "size";
222	7	std::string sizestr = rollmode.substr(sizeflag.size(), rollmode.size() - sizeflag.size());
223	7	if (sizestr.size() != 0) {
224	7	char* end = nullptr;
225	7	errno = 0;
226	7	const char* sizecstr = sizestr.c_str();
227	7	int64_t ret64 = strtoll(sizecstr, &end, 10);
228	7	if ((errno == 0) && (end == sizecstr + strlen(sizecstr))) {
229	7	int32_t retval = static_cast<int32_t>(ret64);
230	7	if (retval == ret64) {
231	7	FLAGS_max_log_size = retval;
232	7	ok = true;
233	7	}
234	7	}
235	7	}
236	7	} else {
237	0	ok = false;
238	0	}
239	7	if (!ok) {
240	0	std::cerr << "sys_log_roll_mode needs to be TIME-DAY, TIME-HOUR, SIZE-MB-nnn" << std::endl;
241	0	return false;
242	0	}
243
244		// set verbose modules.
245	7	FLAGS_v = config::sys_log_verbose_flags_v;
246	7	std::vector<std::string>& verbose_modules = config::sys_log_verbose_modules;
247	7	int32_t vlog_level = config::sys_log_verbose_level;
248	7	for (size_t i = 0; i < verbose_modules.size(); i++) {
249	0	if (verbose_modules[i].size() != 0) {
250	0	google::SetVLOGLevel(verbose_modules[i].c_str(), vlog_level);
251	0	}
252	0	}
253
254	7	if (log_to_console) {
255		// Add prefix if log output to stderr
256	0	if (config::sys_log_enable_custom_date_time_format) {
257	0	google::InitGoogleLogging(basename, &custom_prefix<true, true>);
258	0	} else {
259	0	google::InitGoogleLogging(basename, &custom_prefix<true, false>);
260	0	}
261	7	} else {
262	7	if (config::sys_log_enable_custom_date_time_format) {
263	0	google::InitGoogleLogging(basename, &custom_prefix<false, true>);
264	7	} else {
265	7	google::InitGoogleLogging(basename);
266	7	}
267	7	}
268
269	7	logging_initialized = true;
270
271	7	return true;
272	7	}
273
274	0	void shutdown_logging() {
275	0	std::lock_guard<std::mutex> logging_lock(logging_mutex);
276	0	google::RemoveLogSink(&stdout_log_sink);
277	0	google::ShutdownGoogleLogging();
278	0	}
279
280	0	void update_logging(const std::string& name, const std::string& value) {
281	0	if ("sys_log_level" == name) {
282	0	if (iequals(value, "INFO")) {
283	0	FLAGS_minloglevel = 0;
284	0	} else if (iequals(value, "WARNING")) {
285	0	FLAGS_minloglevel = 1;
286	0	} else if (iequals(value, "ERROR")) {
287	0	FLAGS_minloglevel = 2;
288	0	} else if (iequals(value, "FATAL")) {
289	0	FLAGS_minloglevel = 3;
290	0	} else {
291		LOG(WARNING) << "update sys_log_level failed, need to be INFO, WARNING, ERROR, FATAL";
292	0	}
293	0	}
294	0	}
295
296		#include "common/compile_check_avoid_end.h"
297		} // namespace doris