[System logs]: Eliminate duplicate log messages and attempt rotation more frequently (#520)
* Rename 'ACSFileFormat' -> 'SONiCFileFormat' * Rename '00-acs.conf' -> '00-sonic.conf' * Add logrotate.d and systemd-journald config files to image * Log all SONiC process messages to /var/log/syslog; prevent duplicate logging to /var/log/messages * Do not redirect cron and daemon logs to their own files, let them log to /var/log/syslog * Log all teamd messages to /var/log/teamd.log; Add more SONiC program names to SONiC rules clause * Remove duplicate code by condensing quagga programs into a list; Fix teamd log rule * Kernel and LPR messages no longer getting duplicated to their own log files * Now calling logrotate every minute via cron job * Need full path to logrotate in cron job * Add '.log' suffix to wildcards, otherwise logrotate will rotate already-rotated logs (e.g., bgpd.log.1.1.1.1.1...) * Add microsecond granularity to syslog messages * Don't overwrite system crontab, instead, install additional logrotate crontab file into /etc/cron.d * Removed incomplete concept of per-process SONiC logs. We can revisit again later
This commit is contained in:
Joe LeVeque
lguohan
parent
7dc1045c26
commit
6c202919b3
@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
|
||||
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
|
||||
|
||||
# Define a custom template
|
||||
$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate ACSFileFormat
|
||||
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate SONiCFileFormat
|
||||
|
||||
#
|
||||
# Set the default permissions for all log files.
|
||||
|
||||
@ -88,6 +88,9 @@ sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
|
||||
sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/sonic-device-data_*.deb || \
|
||||
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
|
||||
|
||||
# Copy crontabs
|
||||
sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/
|
||||
|
||||
# Copy NTP configuration files and templates
|
||||
sudo cp $IMAGE_CONFIGS/ntp/ntp-config.service $FILESYSTEM_ROOT/etc/systemd/system/
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ntp-config.service
|
||||
@ -101,6 +104,12 @@ sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.sh $FILESYSTEM_ROOT/usr/bin/
|
||||
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT/usr/share/sonic/templates/
|
||||
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/
|
||||
|
||||
# Copy logrotate.d configuration files
|
||||
sudo cp -f $IMAGE_CONFIGS/logrotate.d/rsyslog $FILESYSTEM_ROOT/etc/logrotate.d/
|
||||
|
||||
# Copy systemd-journald configuration files
|
||||
sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/
|
||||
|
||||
# Copy interfaces configuration files and templates
|
||||
sudo cp $IMAGE_CONFIGS/interfaces/interfaces-config.service $FILESYSTEM_ROOT/etc/systemd/system/
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable interfaces-config.service
|
||||
|
||||
3
files/image_config/cron.d/logrotate
Normal file
3
files/image_config/cron.d/logrotate
Normal file
@ -0,0 +1,3 @@
|
||||
# Attempt to rotate system logs once per minute
|
||||
* * * * * root /usr/sbin/logrotate -f /etc/logrotate.d/rsyslog
|
||||
|
||||
43
files/image_config/logrotate.d/rsyslog
Normal file
43
files/image_config/logrotate.d/rsyslog
Normal file
@ -0,0 +1,43 @@
|
||||
/var/log/syslog
|
||||
/var/log/quagga/*.log
|
||||
/var/log/teamd.log
|
||||
{
|
||||
rotate 7
|
||||
# Removed 'daily' interval, as we now call logrotate more frequently via cron
|
||||
# and we want to check these logs every time
|
||||
# daily
|
||||
size 100M
|
||||
missingok
|
||||
notifempty
|
||||
compress
|
||||
delaycompress
|
||||
postrotate
|
||||
invoke-rc.d rsyslog rotate > /dev/null
|
||||
endscript
|
||||
}
|
||||
/var/log/mail.info
|
||||
/var/log/mail.warn
|
||||
/var/log/mail.err
|
||||
/var/log/mail.log
|
||||
/var/log/daemon.log
|
||||
/var/log/kern.log
|
||||
/var/log/auth.log
|
||||
/var/log/user.log
|
||||
/var/log/lpr.log
|
||||
/var/log/cron.log
|
||||
/var/log/debug
|
||||
/var/log/messages
|
||||
{
|
||||
rotate 4
|
||||
weekly
|
||||
size 100M
|
||||
missingok
|
||||
notifempty
|
||||
compress
|
||||
delaycompress
|
||||
sharedscripts
|
||||
postrotate
|
||||
invoke-rc.d rsyslog rotate > /dev/null
|
||||
endscript
|
||||
}
|
||||
|
||||
@ -42,8 +42,8 @@ $UDPServerRun 514
|
||||
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
|
||||
|
||||
# Define a custom template
|
||||
$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate ACSFileFormat
|
||||
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate SONiCFileFormat
|
||||
|
||||
#
|
||||
# Set the default permissions for all log files.
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
## Quagga rules
|
||||
|
||||
if $programname == "zebra" then {
|
||||
/var/log/quagga/zebra.log
|
||||
stop
|
||||
}
|
||||
|
||||
if $programname == "bgpd" then {
|
||||
/var/log/quagga/bgpd.log
|
||||
stop
|
||||
}
|
||||
|
||||
if $programname == "quagga" then {
|
||||
/var/log/quagga/zebra.log
|
||||
stop
|
||||
}
|
||||
|
||||
if $programname == "watchquagga" then {
|
||||
/var/log/quagga/zebra.log
|
||||
stop
|
||||
}
|
||||
|
||||
## Platform modules rules
|
||||
if $programname == "platform-modules" then {
|
||||
/var/log/syslog
|
||||
stop
|
||||
}
|
||||
|
||||
## Sensord rules
|
||||
if $programname == "sensord" then {
|
||||
/var/log/syslog
|
||||
stop
|
||||
}
|
||||
|
||||
## Sswsyncd rules
|
||||
if $programname == "sswsyncd" then {
|
||||
/var/log/syslog
|
||||
stop
|
||||
}
|
||||
|
||||
## Ansible rules
|
||||
if $programname startswith "ansible" then {
|
||||
/var/log/messages
|
||||
stop
|
||||
}
|
||||
|
||||
22
files/image_config/rsyslog/rsyslog.d/00-sonic.conf
Normal file
22
files/image_config/rsyslog/rsyslog.d/00-sonic.conf
Normal file
@ -0,0 +1,22 @@
|
||||
## Quagga rules
|
||||
|
||||
if $programname == ["quagga",
|
||||
"watchquagga",
|
||||
"zebra"]
|
||||
then {
|
||||
/var/log/quagga/zebra.log
|
||||
stop
|
||||
}
|
||||
|
||||
if $programname == "bgpd" then {
|
||||
/var/log/quagga/bgpd.log
|
||||
stop
|
||||
}
|
||||
|
||||
## Teamd rules
|
||||
|
||||
if $programname contains "teamd_" then {
|
||||
/var/log/teamd.log
|
||||
stop
|
||||
}
|
||||
|
||||
@ -3,11 +3,13 @@
|
||||
#
|
||||
auth,authpriv.* /var/log/auth.log
|
||||
*.*;auth,authpriv.none -/var/log/syslog
|
||||
cron.* /var/log/cron.log
|
||||
daemon.* -/var/log/daemon.log
|
||||
kern.* -/var/log/kern.log
|
||||
kern.* -/var/persist/log/kern.log
|
||||
lpr.* -/var/log/lpr.log
|
||||
# Do not redirect cron, daemon, kernel or lpr logs to
|
||||
# their own files. Let them log to /var/log/syslog
|
||||
#cron.* /var/log/cron.log
|
||||
#daemon.* -/var/log/daemon.log
|
||||
#kern.* -/var/log/kern.log
|
||||
#kern.* -/var/persist/log/kern.log
|
||||
#lpr.* -/var/log/lpr.log
|
||||
mail.* -/var/log/mail.log
|
||||
user.* -/var/log/user.log
|
||||
|
||||
@ -32,10 +34,13 @@ news.notice -/var/log/news/news.notice
|
||||
*.=debug;\
|
||||
auth,authpriv.none;\
|
||||
news.none;mail.none -/var/log/debug
|
||||
*.=info;*.=notice;*.=warn;\
|
||||
auth,authpriv.none;\
|
||||
cron,daemon.none;\
|
||||
mail,news.none -/var/log/messages
|
||||
#
|
||||
# Removed as duplicates:
|
||||
#*.=info;*.=notice;*.=warn;\
|
||||
# auth,authpriv.none;\
|
||||
# cron,daemon.none;\
|
||||
# mail,news.none -/var/log/messages
|
||||
#
|
||||
*.=crit;*.=alert;*.=emerg -/var/persist/log/alarms
|
||||
#
|
||||
# Emergencies are sent to everybody logged in.
|
||||
|
||||
36
files/image_config/systemd/journald.conf
Normal file
36
files/image_config/systemd/journald.conf
Normal file
@ -0,0 +1,36 @@
|
||||
# This file is part of systemd.
|
||||
#
|
||||
# systemd is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Lesser General Public License as published by
|
||||
# the Free Software Foundation; either version 2.1 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# See journald.conf(5) for details
|
||||
|
||||
[Journal]
|
||||
#Storage=auto
|
||||
#Compress=yes
|
||||
#Seal=yes
|
||||
#SplitMode=uid
|
||||
#SyncIntervalSec=5m
|
||||
#RateLimitInterval=30s
|
||||
#RateLimitBurst=1000
|
||||
SystemMaxUse=50MB
|
||||
#SystemKeepFree=
|
||||
#SystemMaxFileSize=
|
||||
RuntimeMaxUse=50MB
|
||||
#RuntimeKeepFree=
|
||||
#RuntimeMaxFileSize=
|
||||
#MaxRetentionSec=
|
||||
#MaxFileSec=1month
|
||||
#ForwardToSyslog=yes
|
||||
#ForwardToKMsg=no
|
||||
#ForwardToConsole=no
|
||||
#ForwardToWall=yes
|
||||
#TTYPath=/dev/console
|
||||
#MaxLevelStore=debug
|
||||
#MaxLevelSyslog=debug
|
||||
#MaxLevelKMsg=notice
|
||||
#MaxLevelConsole=info
|
||||
#MaxLevelWall=emerg
|
||||
|
||||
@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
|
||||
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
|
||||
|
||||
# Define a custom template
|
||||
$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate ACSFileFormat
|
||||
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
|
||||
$ActionFileDefaultTemplate SONiCFileFormat
|
||||
|
||||
#
|
||||
# Set the default permissions for all log files.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user