[System logs]: Eliminate duplicate log messages and attempt rotation more frequently (#520)

* Rename 'ACSFileFormat' -> 'SONiCFileFormat'

* Rename '00-acs.conf' -> '00-sonic.conf'

* Add logrotate.d and systemd-journald config files to image

* Log all SONiC process messages to /var/log/syslog; prevent duplicate logging to /var/log/messages

* Do not redirect cron and daemon logs to their own files, let them log to /var/log/syslog

* Log all teamd messages to /var/log/teamd.log; Add more SONiC program names to SONiC rules clause

* Remove duplicate code by condensing quagga programs into a list; Fix teamd log rule

* Kernel and LPR messages no longer getting duplicated to their own log files

* Now calling logrotate every minute via cron job

* Need full path to logrotate in cron job

* Add '.log' suffix to wildcards, otherwise logrotate will rotate already-rotated logs (e.g., bgpd.log.1.1.1.1.1...)

* Add microsecond granularity to syslog messages

* Don't overwrite system crontab, instead, install additional logrotate crontab file into /etc/cron.d

* Removed incomplete concept of per-process SONiC logs. We can revisit again later

dockers/docker-base/rsyslog.conf

@@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.

files/build_templates/sonic_debian_extension.j2

@@ -88,6 +88,9 @@ sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
 sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/sonic-device-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# Copy crontabs
+sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/
+
 # Copy NTP configuration files and templates
 sudo cp $IMAGE_CONFIGS/ntp/ntp-config.service $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ntp-config.service
@@ -101,6 +104,12 @@ sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.sh $FILESYSTEM_ROOT/usr/bin/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT/usr/share/sonic/templates/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/
 
+# Copy logrotate.d configuration files
+sudo cp -f $IMAGE_CONFIGS/logrotate.d/rsyslog $FILESYSTEM_ROOT/etc/logrotate.d/
+
+# Copy systemd-journald configuration files
+sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/
+
 # Copy interfaces configuration files and templates
 sudo cp $IMAGE_CONFIGS/interfaces/interfaces-config.service  $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable interfaces-config.service

files/image_config/cron.d/logrotate

@@ -0,0 +1,3 @@
+# Attempt to rotate system logs once per minute
+* * * * * root /usr/sbin/logrotate -f /etc/logrotate.d/rsyslog
+

files/image_config/logrotate.d/rsyslog

@@ -0,0 +1,43 @@
+/var/log/syslog
+/var/log/quagga/*.log
+/var/log/teamd.log
+{
+    rotate 7
+# Removed 'daily' interval, as we now call logrotate more frequently via cron
+# and we want to check these logs every time
+#    daily
+    size 100M
+    missingok
+    notifempty
+    compress
+    delaycompress
+    postrotate
+        invoke-rc.d rsyslog rotate > /dev/null
+    endscript
+}
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/cron.log
+/var/log/debug
+/var/log/messages
+{
+    rotate 4
+    weekly
+    size 100M
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        invoke-rc.d rsyslog rotate > /dev/null
+    endscript
+}
+

files/image_config/rsyslog/rsyslog.conf.j2

@@ -42,8 +42,8 @@ $UDPServerRun 514
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.

files/image_config/rsyslog/rsyslog.d/00-acs.conf

@@ -1,46 +0,0 @@
-## Quagga rules
-
-if $programname == "zebra" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-if $programname == "bgpd" then {
-  /var/log/quagga/bgpd.log
-  stop
-}
-
-if $programname == "quagga" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-if $programname == "watchquagga" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-## Platform modules rules
-if $programname == "platform-modules" then {
-  /var/log/syslog
-  stop
-}
-
-## Sensord rules
-if $programname == "sensord" then {
-  /var/log/syslog
-  stop
-}
-
-## Sswsyncd rules
-if $programname == "sswsyncd" then {
-  /var/log/syslog
-  stop
-}
-
-## Ansible rules
-if $programname startswith "ansible" then {
-  /var/log/messages
-  stop
-}
-

files/image_config/rsyslog/rsyslog.d/00-sonic.conf

@@ -0,0 +1,22 @@
+## Quagga rules
+
+if $programname == ["quagga",
+                    "watchquagga",
+                    "zebra"]
+    then {
+    /var/log/quagga/zebra.log
+    stop
+}
+
+if $programname == "bgpd" then {
+    /var/log/quagga/bgpd.log
+    stop
+}
+
+## Teamd rules
+
+if $programname contains "teamd_" then {
+    /var/log/teamd.log
+    stop
+}
+

files/image_config/rsyslog/rsyslog.d/99-default.conf

@@ -3,11 +3,13 @@
 #
 auth,authpriv.*                 /var/log/auth.log
 *.*;auth,authpriv.none          -/var/log/syslog
-cron.*                          /var/log/cron.log
+# Do not redirect cron, daemon, kernel or lpr logs to
-daemon.*                        -/var/log/daemon.log
+# their own files. Let them log to /var/log/syslog
-kern.*                          -/var/log/kern.log
+#cron.*                          /var/log/cron.log
-kern.*                          -/var/persist/log/kern.log
+#daemon.*                        -/var/log/daemon.log
-lpr.*                           -/var/log/lpr.log
+#kern.*                          -/var/log/kern.log
+#kern.*                          -/var/persist/log/kern.log
+#lpr.*                           -/var/log/lpr.log
 mail.*                          -/var/log/mail.log
 user.*                          -/var/log/user.log
 
@@ -32,10 +34,13 @@ news.notice                     -/var/log/news/news.notice
 *.=debug;\
         auth,authpriv.none;\
         news.none;mail.none     -/var/log/debug
-*.=info;*.=notice;*.=warn;\
+#
-        auth,authpriv.none;\
+# Removed as duplicates:
-        cron,daemon.none;\
+#*.=info;*.=notice;*.=warn;\
-        mail,news.none          -/var/log/messages
+#        auth,authpriv.none;\
+#        cron,daemon.none;\
+#        mail,news.none          -/var/log/messages
+#
 *.=crit;*.=alert;*.=emerg     -/var/persist/log/alarms
 #
 # Emergencies are sent to everybody logged in.

files/image_config/systemd/journald.conf

@@ -0,0 +1,36 @@
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+#
+# See journald.conf(5) for details
+
+[Journal]
+#Storage=auto
+#Compress=yes
+#Seal=yes
+#SplitMode=uid
+#SyncIntervalSec=5m
+#RateLimitInterval=30s
+#RateLimitBurst=1000
+SystemMaxUse=50MB
+#SystemKeepFree=
+#SystemMaxFileSize=
+RuntimeMaxUse=50MB
+#RuntimeKeepFree=
+#RuntimeMaxFileSize=
+#MaxRetentionSec=
+#MaxFileSec=1month
+#ForwardToSyslog=yes
+#ForwardToKMsg=no
+#ForwardToConsole=no
+#ForwardToWall=yes
+#TTYPath=/dev/console
+#MaxLevelStore=debug
+#MaxLevelSyslog=debug
+#MaxLevelKMsg=notice
+#MaxLevelConsole=info
+#MaxLevelWall=emerg
+

platform/p4/docker-sonic-p4/rsyslog.conf

@@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.