[System logs]: Improvements to prevent filling /var/log partition (#865)

- Force log rotation at size thresholds only (no longer also rotating logs daily), allowing for more consistent archived log size - Eliminate remaining duplicate log messages - Cron facility now only logs to cron.log (was also logging to syslog) - Debug, mail, news and user log facilities only log to syslog; no longer creating separate log files for these facilities - Cron job that calls logrotate every minute now uses the main /etc/logrotate.conf file so as to check/rotate all logs every minute, not just the logs specified in the rsyslog file. Also redirecting output of this command to /dev/null to prevent "(CRON) info (No MTA installed, discarding output)" messages in cron.log due to lack of a mail service - Delete archive files based on remaining /var/log partition space. Note that this solution currently requires a minimum /var/log partition size of 32MB to function correctly - Update sonic-sairedis and sonic-swss submodules to incorporate recording file name changes - Add .screen file to .gitignore (unrelated)
2017-08-10 16:24:57 -07:00 · 2017-08-10 16:24:57 -07:00 · 9d5adb993a
commit 9d5adb993a
parent 847a948069
12 changed files with 182 additions and 113 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,5 +1,6 @@
 # Build system related
 .platform
+.screen

 # Build artifacts
 fsroot/
--- a/files/build_templates/sonic_debian_extension.j2
+++ b/files/build_templates/sonic_debian_extension.j2
@ -106,7 +106,7 @@ sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT/usr/share/sonic/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/

 # Copy logrotate.d configuration files
-sudo cp -f $IMAGE_CONFIGS/logrotate.d/rsyslog $FILESYSTEM_ROOT/etc/logrotate.d/
+sudo cp -f $IMAGE_CONFIGS/logrotate/logrotate.d/* $FILESYSTEM_ROOT/etc/logrotate.d/

 # Copy systemd-journald configuration files
 sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/
--- a/files/image_config/cron.d/logrotate
+++ b/files/image_config/cron.d/logrotate
@ -1,2 +1,2 @@
 # Attempt to rotate system logs once per minute
-* * * * * root /usr/sbin/logrotate /etc/logrotate.d/rsyslog
+* * * * * root /usr/sbin/logrotate /etc/logrotate.conf > /dev/null 2>&1
--- a/files/image_config/logrotate.d/rsyslog
+++ b/files/image_config/logrotate.d/rsyslog
@ -1,62 +0,0 @@
-/var/log/syslog
-/var/log/quagga/*.log
-/var/log/teamd.log
-{
-    rotate 7
-    daily
-    maxsize 100M
-    missingok
-    notifempty
-    compress
-    delaycompress
-    postrotate
-        # Calling kill directly instead of 'service rsyslog rotate >/dev/null' due
-        # to bug in init-system-helpers. Bug has apparently been fixed in v1.47.
-        # However, Debian Jessie is still using v1.22.
-        # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
-        kill -HUP $(cat /var/run/rsyslogd.pid)
-    endscript
-}
-/var/log/swss/*.rec
-{
-    rotate 7
-    daily
-    maxsize 20M
-    missingok
-    notifempty
-    compress
-    delaycompress
-    sharedscripts
-    postrotate
-        pgrep -x orchagent | xargs /bin/kill -HUP 2>/dev/null || true
-    endscript
-}
-/var/log/mail.info
-/var/log/mail.warn
-/var/log/mail.err
-/var/log/mail.log
-/var/log/daemon.log
-/var/log/kern.log
-/var/log/auth.log
-/var/log/user.log
-/var/log/lpr.log
-/var/log/cron.log
-/var/log/debug
-/var/log/messages
-{
-    rotate 4
-    daily
-    maxsize 50M
-    missingok
-    notifempty
-    compress
-    delaycompress
-    sharedscripts
-    postrotate
-        # Calling kill directly instead of 'service rsyslog rotate >/dev/null' due
-        # to bug in init-system-helpers. Bug has apparently been fixed in v1.47.
-        # However, Debian Jessie is still using v1.22.
-        # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
-        kill -HUP $(cat /var/run/rsyslogd.pid)
-    endscript
-}
--- a/files/image_config/logrotate/logrotate.conf
+++ b/files/image_config/logrotate/logrotate.conf
@ -0,0 +1,32 @@
+# see "man logrotate" for details
+# Rotate log files daily by default
+daily
+
+# Keep 4 days worth of backlogs by default
+rotate 4
+
+# create new (empty) log files after rotating old ones
+create
+
+# uncomment this if you want your log files compressed
+#compress
+
+# packages drop log rotation information into this directory
+include /etc/logrotate.d
+
+# no packages own wtmp, or btmp -- we'll rotate them here
+/var/log/wtmp {
+    missingok
+    size 100k
+    create 0664 root utmp
+    rotate 1
+}
+
+/var/log/btmp {
+    missingok
+    size 100k
+    create 0660 root utmp
+    rotate 1
+}
+
+# system-specific logs may be configured here
--- a/files/image_config/logrotate/logrotate.d/apt
+++ b/files/image_config/logrotate/logrotate.d/apt
@ -0,0 +1,15 @@
+/var/log/apt/term.log {
+  size 100k
+  rotate 1
+  compress
+  missingok
+  notifempty
+}
+
+/var/log/apt/history.log {
+  size 100k
+  rotate 1
+  compress
+  missingok
+  notifempty
+}
--- a/files/image_config/logrotate/logrotate.d/dpkg
+++ b/files/image_config/logrotate/logrotate.d/dpkg
@ -0,0 +1,18 @@
+/var/log/dpkg.log {
+        size 100k
+        rotate 1
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 644 root root
+}
+/var/log/alternatives.log {
+        size 100k
+        rotate 1
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 644 root root
+}
--- a/files/image_config/logrotate/logrotate.d/monit
+++ b/files/image_config/logrotate/logrotate.d/monit
@ -0,0 +1,12 @@
+/var/log/monit.log {
+        size 100k
+        rotate 1
+        missingok
+        create 640 root adm
+        notifempty
+        compress
+        delaycompress
+        postrotate
+                invoke-rc.d monit reload > /dev/null
+        endscript
+}
--- a/files/image_config/logrotate/logrotate.d/rsyslog
+++ b/files/image_config/logrotate/logrotate.d/rsyslog
@ -0,0 +1,96 @@
+# These logs should no longer get created. However, in case they do get created,
+# we should keep them to a small size and rotate them also.
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/debug
+/var/log/messages
+{
+    size 10k
+    rotate 1
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        # calling kill directly instead of 'service rsyslog rotate >/dev/null' due
+        # to bug in init-system-helpers. bug has apparently been fixed in v1.47.
+        # however, debian jessie is still using v1.22.
+        # see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
+        kill -hup $(cat /var/run/rsyslogd.pid)
+    endscript
+}
+
+/var/log/auth.log
+/var/log/cron.log
+/var/log/syslog
+/var/log/teamd.log
+/var/log/quagga/bgpd.log
+/var/log/quagga/zebra.log
+/var/log/swss/sairedis.rec
+/var/log/swss/swss.rec
+{
+    size 1M
+    rotate 5000
+    missingok
+    notifempty
+    compress
+    delaycompress
+    nosharedscripts
+    prerotate
+        # Adjust NUM_LOGS_TO_ROTATE to reflect number of log files that trigger this block specified above
+        NUM_LOGS_TO_ROTATE=8
+
+        # Adjust LOG_FILE_ROTATE_SIZE_KB to reflect the "size" parameter specified above, in kB
+        LOG_FILE_ROTATE_SIZE_KB=1024
+
+        # Reserve space for btmp, wtmp, dpkg.log, monit.log, etc., as well as logs that
+        # should be disabled, just in case they get created and rotated
+        RESERVED_SPACE_KB=4096
+
+        VAR_LOG_SIZE_KB=$(df -k /var/log | sed -n 2p | awk '{ print $2 }')
+
+        # Limit usable space to 95% of the partition minus the reserved space for other logs
+        USABLE_SPACE_KB=$(( (VAR_LOG_SIZE_KB * 95 / 100) - RESERVED_SPACE_KB))
+
+        # Set our threshold so as to maintain enough space to write all logs from empty to full
+        # Most likely, some logs will have non-zero size when this is called, so this errs on the side
+        # of caution, giving us a bit of a cushion if a log grows quickly and passes its rotation size
+        THRESHOLD_KB=$((USABLE_SPACE_KB - (NUM_LOGS_TO_ROTATE * LOG_FILE_ROTATE_SIZE_KB * 2)))
+
+        while true; do
+            USED_KB=$(du -s /var/log | awk '{ print $1; }')
+
+            if [ $USED_KB -lt $THRESHOLD_KB ]; then
+                break
+            else
+                OLDEST_ARCHIVE_FILE=$(find /var/log -type f -printf '%T+ %p\n' | grep -E '.+\.[0-9]+(\.gz)?$' | sort | head -n 1 | awk '{ print $2; }')
+
+                if [ -z "$OLDEST_ARCHIVE_FILE" ]; then
+                    logger -p syslog.err -t "logrotate" "No archive file to delete -- potential for filling up /var/log partition!"
+                    break
+                fi
+
+                logger -p syslog.info -t "logrotate" "Deleting archive file $OLDEST_ARCHIVE_FILE to free up space"
+                rm -rf "$OLDEST_ARCHIVE_FILE"
+            fi
+        done
+    endscript
+    postrotate
+        if [ $(echo $1 | grep -c "/var/log/swss/") -gt 0 ]; then
+            pgrep -x orchagent | xargs /bin/kill -HUP 2>/dev/null || true
+        else
+            # Calling kill directly instead of 'service rsyslog rotate >/dev/null' due
+            # to bug in init-system-helpers. Bug has apparently been fixed in v1.47.
+            # However, Debian Jessie is still using v1.22.
+            # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
+            kill -HUP $(cat /var/run/rsyslogd.pid)
+        fi
+    endscript
+}
--- a/files/image_config/rsyslog/rsyslog.d/99-default.conf
+++ b/files/image_config/rsyslog/rsyslog.d/99-default.conf
@ -1,61 +1,18 @@
 #
 # First some standard log files.  Log by facility.
 #
+
+# Log all facilities to /var/log/syslog except cron, auth
+# and authpriv. They are noisy - log them to their own files
+*.*;cron,auth,authpriv.none     -/var/log/syslog
 auth,authpriv.*                 /var/log/auth.log
-*.*;auth,authpriv.none          -/var/log/syslog
 cron.*                          /var/log/cron.log
-# Do not redirect daemon, kernel or lpr logs to
-# their own files. Let them log to /var/log/syslog
-#daemon.*                        -/var/log/daemon.log
-#kern.*                          -/var/log/kern.log
-#kern.*                          -/var/persist/log/kern.log
-#lpr.*                           -/var/log/lpr.log
-mail.*                          -/var/log/mail.log
-user.*                          -/var/log/user.log

-#
-# Logging for the mail system.  Split it up so that
-# it is easy to write scripts to parse these files.
-#
-mail.info                       -/var/log/mail.info
-mail.warn                       -/var/log/mail.warn
-mail.err                        /var/log/mail.err
-
-#
-# Logging for INN news system.
-#
-news.crit                       /var/log/news/news.crit
-news.err                        /var/log/news/news.err
-news.notice                     -/var/log/news/news.notice
-
-#
-# Some "catch-all" log files.
-#
-*.=debug;\
-        auth,authpriv.none;\
-        news.none;mail.none     -/var/log/debug
-#
-# Removed as duplicates:
-#*.=info;*.=notice;*.=warn;\
-#        auth,authpriv.none;\
-#        cron,daemon.none;\
-#        mail,news.none          -/var/log/messages
-#
-*.=crit;*.=alert;*.=emerg     -/var/persist/log/alarms
 #
 # Emergencies are sent to everybody logged in.
 #
 *.emerg                         :omusrmsg:*

-#
-# I like to have messages displayed on the console, but only on a virtual
-# console I usually leave idle.
-#
-#daemon,mail.*;\
-#       news.=crit;news.=err;news.=notice;\
-#       *.=debug;*.=info;\
-#       *.=notice;*.=warn       /dev/tty8
-
 # The named pipe /dev/xconsole is for the `xconsole' utility.  To use it,
 # you must invoke `xconsole' with the `-file' option:
 #
--- a/src/sonic-sairedis
+++ b/src/sonic-sairedis
@ -1 +1 @@
-Subproject commit b865912105f2f80ee2516899e6367f863d5f6235
+Subproject commit 7e70b4d63020fc9ea1aa009c264e4effab7bcda0
--- a/src/sonic-swss
+++ b/src/sonic-swss
@ -1 +1 @@
-Subproject commit 27c49a0bf3a71556b553aa30fe2a7c1cdb996142
+Subproject commit ab31a9af7ef2c5ea9fe5f0f89f0a61b3ec0f83bf