From 6c202919b30a04930f794e2a2ee4c4d8a19c2b10 Mon Sep 17 00:00:00 2001
From: Joe LeVeque <jleveque@users.noreply.github.com>
Date: Fri, 21 Apr 2017 08:22:44 -0700
Subject: [PATCH] [System logs]: Eliminate duplicate log messages and attempt
 rotation more frequently (#520)

* Rename 'ACSFileFormat' -> 'SONiCFileFormat'

* Rename '00-acs.conf' -> '00-sonic.conf'

* Add logrotate.d and systemd-journald config files to image

* Log all SONiC process messages to /var/log/syslog; prevent duplicate logging to /var/log/messages

* Do not redirect cron and daemon logs to their own files, let them log to /var/log/syslog

* Log all teamd messages to /var/log/teamd.log; Add more SONiC program names to SONiC rules clause

* Remove duplicate code by condensing quagga programs into a list; Fix teamd log rule

* Kernel and LPR messages no longer getting duplicated to their own log files

* Now calling logrotate every minute via cron job

* Need full path to logrotate in cron job

* Add '.log' suffix to wildcards, otherwise logrotate will rotate already-rotated logs (e.g., bgpd.log.1.1.1.1.1...)

* Add microsecond granularity to syslog messages

* Don't overwrite system crontab, instead, install additional logrotate crontab file into /etc/cron.d

* Removed incomplete concept of per-process SONiC logs. We can revisit again later
---
 dockers/docker-base/rsyslog.conf              |  4 +-
 .../build_templates/sonic_debian_extension.j2 |  9 ++++
 files/image_config/cron.d/logrotate           |  3 ++
 files/image_config/logrotate.d/rsyslog        | 43 +++++++++++++++++
 files/image_config/rsyslog/rsyslog.conf.j2    |  4 +-
 .../rsyslog/rsyslog.d/00-acs.conf             | 46 -------------------
 .../rsyslog/rsyslog.d/00-sonic.conf           | 22 +++++++++
 .../rsyslog/rsyslog.d/99-default.conf         | 23 ++++++----
 files/image_config/systemd/journald.conf      | 36 +++++++++++++++
 platform/p4/docker-sonic-p4/rsyslog.conf      |  4 +-
 10 files changed, 133 insertions(+), 61 deletions(-)
 create mode 100644 files/image_config/cron.d/logrotate
 create mode 100644 files/image_config/logrotate.d/rsyslog
 delete mode 100644 files/image_config/rsyslog/rsyslog.d/00-acs.conf
 create mode 100644 files/image_config/rsyslog/rsyslog.d/00-sonic.conf
 create mode 100644 files/image_config/systemd/journald.conf

diff --git a/dockers/docker-base/rsyslog.conf b/dockers/docker-base/rsyslog.conf
index e03df1f741..1132ad55ec 100644
--- a/dockers/docker-base/rsyslog.conf
+++ b/dockers/docker-base/rsyslog.conf
@@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.
diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2
index 77fcd1f386..16a25d2c3e 100644
--- a/files/build_templates/sonic_debian_extension.j2
+++ b/files/build_templates/sonic_debian_extension.j2
@@ -88,6 +88,9 @@ sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
 sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/sonic-device-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# Copy crontabs
+sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/
+
 # Copy NTP configuration files and templates
 sudo cp $IMAGE_CONFIGS/ntp/ntp-config.service $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ntp-config.service
@@ -101,6 +104,12 @@ sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.sh $FILESYSTEM_ROOT/usr/bin/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT/usr/share/sonic/templates/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/
 
+# Copy logrotate.d configuration files
+sudo cp -f $IMAGE_CONFIGS/logrotate.d/rsyslog $FILESYSTEM_ROOT/etc/logrotate.d/
+
+# Copy systemd-journald configuration files
+sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/
+
 # Copy interfaces configuration files and templates
 sudo cp $IMAGE_CONFIGS/interfaces/interfaces-config.service  $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable interfaces-config.service
diff --git a/files/image_config/cron.d/logrotate b/files/image_config/cron.d/logrotate
new file mode 100644
index 0000000000..e7813aa1f5
--- /dev/null
+++ b/files/image_config/cron.d/logrotate
@@ -0,0 +1,3 @@
+# Attempt to rotate system logs once per minute
+* * * * * root /usr/sbin/logrotate -f /etc/logrotate.d/rsyslog
+
diff --git a/files/image_config/logrotate.d/rsyslog b/files/image_config/logrotate.d/rsyslog
new file mode 100644
index 0000000000..c09604187f
--- /dev/null
+++ b/files/image_config/logrotate.d/rsyslog
@@ -0,0 +1,43 @@
+/var/log/syslog
+/var/log/quagga/*.log
+/var/log/teamd.log
+{
+    rotate 7
+# Removed 'daily' interval, as we now call logrotate more frequently via cron
+# and we want to check these logs every time
+#    daily
+    size 100M
+    missingok
+    notifempty
+    compress
+    delaycompress
+    postrotate
+        invoke-rc.d rsyslog rotate > /dev/null
+    endscript
+}
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/cron.log
+/var/log/debug
+/var/log/messages
+{
+    rotate 4
+    weekly
+    size 100M
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        invoke-rc.d rsyslog rotate > /dev/null
+    endscript
+}
+
diff --git a/files/image_config/rsyslog/rsyslog.conf.j2 b/files/image_config/rsyslog/rsyslog.conf.j2
index e00565f056..008f9f10e6 100644
--- a/files/image_config/rsyslog/rsyslog.conf.j2
+++ b/files/image_config/rsyslog/rsyslog.conf.j2
@@ -42,8 +42,8 @@ $UDPServerRun 514
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.
diff --git a/files/image_config/rsyslog/rsyslog.d/00-acs.conf b/files/image_config/rsyslog/rsyslog.d/00-acs.conf
deleted file mode 100644
index ec8ce919f4..0000000000
--- a/files/image_config/rsyslog/rsyslog.d/00-acs.conf
+++ /dev/null
@@ -1,46 +0,0 @@
-## Quagga rules
-
-if $programname == "zebra" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-if $programname == "bgpd" then {
-  /var/log/quagga/bgpd.log
-  stop
-}
-
-if $programname == "quagga" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-if $programname == "watchquagga" then {
-  /var/log/quagga/zebra.log
-  stop
-}
-
-## Platform modules rules
-if $programname == "platform-modules" then {
-  /var/log/syslog
-  stop
-}
-
-## Sensord rules
-if $programname == "sensord" then {
-  /var/log/syslog
-  stop
-}
-
-## Sswsyncd rules
-if $programname == "sswsyncd" then {
-  /var/log/syslog
-  stop
-}
-
-## Ansible rules
-if $programname startswith "ansible" then {
-  /var/log/messages
-  stop
-}
-
diff --git a/files/image_config/rsyslog/rsyslog.d/00-sonic.conf b/files/image_config/rsyslog/rsyslog.d/00-sonic.conf
new file mode 100644
index 0000000000..20d18e5d22
--- /dev/null
+++ b/files/image_config/rsyslog/rsyslog.d/00-sonic.conf
@@ -0,0 +1,22 @@
+## Quagga rules
+
+if $programname == ["quagga",
+                    "watchquagga",
+                    "zebra"]
+    then {
+    /var/log/quagga/zebra.log
+    stop
+}
+
+if $programname == "bgpd" then {
+    /var/log/quagga/bgpd.log
+    stop
+}
+
+## Teamd rules
+
+if $programname contains "teamd_" then {
+    /var/log/teamd.log
+    stop
+}
+
diff --git a/files/image_config/rsyslog/rsyslog.d/99-default.conf b/files/image_config/rsyslog/rsyslog.d/99-default.conf
index a26ba7baf8..9b129199b3 100644
--- a/files/image_config/rsyslog/rsyslog.d/99-default.conf
+++ b/files/image_config/rsyslog/rsyslog.d/99-default.conf
@@ -3,11 +3,13 @@
 #
 auth,authpriv.*                 /var/log/auth.log
 *.*;auth,authpriv.none          -/var/log/syslog
-cron.*                          /var/log/cron.log
-daemon.*                        -/var/log/daemon.log
-kern.*                          -/var/log/kern.log
-kern.*                          -/var/persist/log/kern.log
-lpr.*                           -/var/log/lpr.log
+# Do not redirect cron, daemon, kernel or lpr logs to
+# their own files. Let them log to /var/log/syslog
+#cron.*                          /var/log/cron.log
+#daemon.*                        -/var/log/daemon.log
+#kern.*                          -/var/log/kern.log
+#kern.*                          -/var/persist/log/kern.log
+#lpr.*                           -/var/log/lpr.log
 mail.*                          -/var/log/mail.log
 user.*                          -/var/log/user.log
 
@@ -32,10 +34,13 @@ news.notice                     -/var/log/news/news.notice
 *.=debug;\
         auth,authpriv.none;\
         news.none;mail.none     -/var/log/debug
-*.=info;*.=notice;*.=warn;\
-        auth,authpriv.none;\
-        cron,daemon.none;\
-        mail,news.none          -/var/log/messages
+#
+# Removed as duplicates:
+#*.=info;*.=notice;*.=warn;\
+#        auth,authpriv.none;\
+#        cron,daemon.none;\
+#        mail,news.none          -/var/log/messages
+#
 *.=crit;*.=alert;*.=emerg     -/var/persist/log/alarms
 #
 # Emergencies are sent to everybody logged in.
diff --git a/files/image_config/systemd/journald.conf b/files/image_config/systemd/journald.conf
new file mode 100644
index 0000000000..fc6c4a4e7a
--- /dev/null
+++ b/files/image_config/systemd/journald.conf
@@ -0,0 +1,36 @@
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+#
+# See journald.conf(5) for details
+
+[Journal]
+#Storage=auto
+#Compress=yes
+#Seal=yes
+#SplitMode=uid
+#SyncIntervalSec=5m
+#RateLimitInterval=30s
+#RateLimitBurst=1000
+SystemMaxUse=50MB
+#SystemKeepFree=
+#SystemMaxFileSize=
+RuntimeMaxUse=50MB
+#RuntimeKeepFree=
+#RuntimeMaxFileSize=
+#MaxRetentionSec=
+#MaxFileSec=1month
+#ForwardToSyslog=yes
+#ForwardToKMsg=no
+#ForwardToConsole=no
+#ForwardToWall=yes
+#TTYPath=/dev/console
+#MaxLevelStore=debug
+#MaxLevelSyslog=debug
+#MaxLevelKMsg=notice
+#MaxLevelConsole=info
+#MaxLevelWall=emerg
+
diff --git a/platform/p4/docker-sonic-p4/rsyslog.conf b/platform/p4/docker-sonic-p4/rsyslog.conf
index d82ea610d6..e8f42bdabe 100644
--- a/platform/p4/docker-sonic-p4/rsyslog.conf
+++ b/platform/p4/docker-sonic-p4/rsyslog.conf
@@ -39,8 +39,8 @@ $ModLoad imuxsock # provides support for local system logging
 #$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
 # Define a custom template
-$template ACSFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate ACSFileFormat
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$ActionFileDefaultTemplate SONiCFileFormat
 
 #
 # Set the default permissions for all log files.