[monit] Build from source and patch to use MemAvailable value if available on system (#3875)

.gitignore

@@ -60,6 +60,9 @@ src/lldpd/*
 !src/lldpd/patch/
 src/lm-sensors/*
 !src/lm-sensors/Makefile
+src/monit/*
+!src/monit/Makefile
+!src/monit/patch/
 src/mpdecimal/*
 !src/mpdecimal/Makefile
 src/python-click/*

build_debian.sh

@@ -240,7 +240,6 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
     openssh-server          \
     python                  \
     python-setuptools       \
-    monit                   \
     python-apt              \
     traceroute              \
     iputils-ping            \
@@ -348,10 +347,6 @@ EOF
 sudo sed -i 's/^ListenAddress ::/#ListenAddress ::/' $FILESYSTEM_ROOT/etc/ssh/sshd_config
 sudo sed -i 's/^#ListenAddress 0.0.0.0/ListenAddress 0.0.0.0/' $FILESYSTEM_ROOT/etc/ssh/sshd_config
 
-## Config monit
-sudo cp files/image_config/monit/monitrc $FILESYSTEM_ROOT/etc/monit/
-sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/monitrc
-
 ## Config sysctl
 sudo mkdir -p $FILESYSTEM_ROOT/var/core
 sudo augtool --autosave "

files/build_templates/sonic_debian_extension.j2

@@ -172,6 +172,14 @@ sudo sed -i -e '/^passwd/s/ tacplus//' $FILESYSTEM_ROOT/etc/nsswitch.conf
 sudo DEBIAN_FRONTEND=noninteractive dpkg --root=$FILESYSTEM_ROOT -i $debs_path/kdump-tools_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=truechroot $FILESYSTEM_ROOT apt-get -q --no-install-suggests --no-install-recommends --force-no install
 
+# Install custom-built monit package and SONiC configuration files
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/monit_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+sudo cp $IMAGE_CONFIGS/monit/monitrc $FILESYSTEM_ROOT/etc/monit/
+sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/monitrc
+sudo cp $IMAGE_CONFIGS/monit/conf.d/* $FILESYSTEM_ROOT/etc/monit/conf.d/
+sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/conf.d/*
+
 # Copy crontabs
 sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/
 

files/image_config/monit/conf.d/sonic-host

@@ -0,0 +1,22 @@
+###############################################################################
+## Monit configuration for SONiC host OS
+##
+## This includes system-level monitoring as well as processes which
+## run in the host OS (i.e., not inside a Docker container)
+###############################################################################
+
+check filesystem root-overlay with path /
+    if space usage > 90% for 5 times within 10 cycles then alert
+
+check filesystem var-log with path /var/log
+    if space usage > 90% for 5 times within 10 cycles then alert
+
+check system $HOST
+    if memory usage > 90% for 5 times within 10 cycles then alert
+    if cpu usage (user) > 90% for 5 times within 10 cycles then alert
+    if cpu usage (system) > 90% for 5 times within 10 cycles then alert
+
+check process rsyslog with pidfile /var/run/rsyslogd.pid
+    start program = "/bin/systemctl start rsyslog.service"
+    stop program = "/bin/systemctl stop rsyslog.service"
+    if totalmem > 800 MB for 5 times within 10 cycles then restart

files/image_config/monit/monitrc

@@ -24,8 +24,7 @@
 ## Set syslog logging. If you want to log to a standalone log file instead,
 ## specify the full path to the log file
 #
-# set logfile /var/log/monit.log
+  set logfile syslog
-set logfile syslog
 #
 #
 ## Set the location of the Monit lock file which stores the process id of the
@@ -153,8 +152,8 @@ set logfile syslog
 ## commands to a running Monit daemon. See the Monit Wiki if you want to
 ## enable SSL for the HTTP interface.
 #
-set httpd unixsocket /var/run/monit.sock and
+  set httpd unixsocket /var/run/monit.sock and
-     allow localhost        # allow localhost to connect to the server and
+      allow localhost        # allow localhost to connect to the server
 #
 ###############################################################################
 ## Services
@@ -294,15 +293,3 @@ set httpd unixsocket /var/run/monit.sock and
    include /etc/monit/conf.d/*
    include /etc/monit/conf-enabled/*
 #
-check filesystem root-overlay with path /
-  if space usage > 90% for 5 times within 10 cycles then alert
-check filesystem var-log with path /var/log
-  if space usage > 90% for 5 times within 10 cycles then alert
-check system $HOST
-  if memory usage > 90% for 5 times within 10 cycles then alert
-  if cpu usage (user) > 90% for 5 times within 10 cycles then alert
-  if cpu usage (system) > 90% for 5 times within 10 cycles then alert
-check process rsyslog with pidfile /var/run/rsyslogd.pid
-  start program = "/bin/systemctl start rsyslog.service"
-  stop program = "/bin/systemctl stop rsyslog.service"
-  if totalmem > 800 MB for 5 times within 10 cycles then restart

rules/monit.mk

@@ -0,0 +1,14 @@
+# monit package
+
+MONIT_VERSION = 5.20.0-6
+
+export MONIT_VERSION
+
+MONIT = monit_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb
+$(MONIT)_SRC_PATH = $(SRC_PATH)/monit
+SONIC_MAKE_DEBS += $(MONIT)
+
+SONIC_STRETCH_DEBS += $(MONIT)
+
+MONIT_DBG = monit-dbgsym_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(MONIT),$(MONIT_DBG)))

slave.mk

@@ -617,7 +617,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(IFUPDOWN2) \
                 $(KDUMP_TOOLS) \
                 $(LIBPAM_TACPLUS) \
-                $(LIBNSS_TACPLUS)) \
+                $(LIBNSS_TACPLUS) \
+                $(MONIT)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
 	$(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(DEBS_PATH)/,$(SONIC_ZTP))) \

src/monit/Makefile

@@ -0,0 +1,33 @@
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -e
+
+MAIN_TARGET = monit_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb
+DERIVED_TARGETS = monit-dbgsym_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	# Remove any stale files
+	rm -rf ./monit
+
+	# Clone monit repo
+	git clone https://salsa.debian.org/sk-guest/monit.git
+	pushd ./monit
+
+	# Reset HEAD to the commit of the proper tag
+	# NOTE: Using "git checkout <tag_name>" here detaches our HEAD,
+	# which stg doesn't like, so we use this method instead
+	# NOTE: For some reason, tags in the Debian monit repo are prefixed with "1%"
+	git reset --hard debian/1\%$(MONIT_VERSION)
+
+	# Apply patches
+	stg init
+	stg import -s ../patch/series
+
+	# Build source and Debian packages
+	dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
+	popd
+
+	# Move the newly-built .deb packages to the destination directory
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

src/monit/patch/0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch

@@ -0,0 +1,70 @@
+From c392362c9c1d57256b7e8ab7c77926824677fd73 Mon Sep 17 00:00:00 2001
+From: Joe LeVeque <jolevequ@microsoft.com>
+Date: Tue, 19 Nov 2019 01:51:13 +0000
+Subject: [PATCH] [used_system_memory_sysdep] Use 'MemAvailable' value if
+ available
+
+---
+ src/process/sysdep_LINUX.c | 35 +++++++++++++++++++++++------------
+ 1 file changed, 23 insertions(+), 12 deletions(-)
+
+diff --git a/src/process/sysdep_LINUX.c b/src/process/sysdep_LINUX.c
+index 0d18f85..221e785 100644
+--- a/src/process/sysdep_LINUX.c
++++ b/src/process/sysdep_LINUX.c
+@@ -335,6 +335,7 @@ int getloadavg_sysdep(double *loadv, int nelem) {
+ boolean_t used_system_memory_sysdep(SystemInfo_T *si) {
+         char          *ptr;
+         char           buf[2048];
++        unsigned long  mem_available = 0UL;
+         unsigned long  mem_free = 0UL;
+         unsigned long  buffers = 0UL;
+         unsigned long  cached = 0UL;
+@@ -343,22 +344,32 @@ boolean_t used_system_memory_sysdep(SystemInfo_T *si) {
+         unsigned long  swap_free = 0UL;
+ 
+         if (! file_readProc(buf, sizeof(buf), "meminfo", -1, NULL)) {
+-                LogError("system statistic error -- cannot get real memory free amount\n");
++                LogError("system statistic error -- cannot read /proc/meminfo\n");
+                 goto error;
+         }
+ 
+-        /* Memory */
+-        if (! (ptr = strstr(buf, "MemFree:")) || sscanf(ptr + 8, "%ld", &mem_free) != 1) {
+-                LogError("system statistic error -- cannot get real memory free amount\n");
+-                goto error;
++        /*
++         * Memory
++         *
++         * First, check if the "MemAvailable" value is available on this system. If it is, we will
++         * use it. Otherwise we will attempt to calculate the amount of available memory ourself.
++         */
++        if ((ptr = strstr(buf, "MemAvailable:")) && sscanf(ptr + 13, "%ld", &mem_available) == 1) {
++                si->total_mem = systeminfo.mem_max - (uint64_t)mem_available * 1024;
++        } else {
++                DEBUG("'MemAvailable' value not available on this system. Attempting to calculate available memory manually...\n");
++                if (! (ptr = strstr(buf, "MemFree:")) || sscanf(ptr + 8, "%ld", &mem_free) != 1) {
++                        LogError("system statistic error -- cannot get real memory free amount\n");
++                        goto error;
++                }
++                if (! (ptr = strstr(buf, "Buffers:")) || sscanf(ptr + 8, "%ld", &buffers) != 1)
++                        DEBUG("system statistic error -- cannot get real memory buffers amount\n");
++                if (! (ptr = strstr(buf, "Cached:")) || sscanf(ptr + 7, "%ld", &cached) != 1)
++                        DEBUG("system statistic error -- cannot get real memory cache amount\n");
++                if (! (ptr = strstr(buf, "SReclaimable:")) || sscanf(ptr + 13, "%ld", &slabreclaimable) != 1)
++                        DEBUG("system statistic error -- cannot get slab reclaimable memory amount\n");
++                si->total_mem = systeminfo.mem_max - (uint64_t)(mem_free + buffers + cached + slabreclaimable) * 1024;
+         }
+-        if (! (ptr = strstr(buf, "Buffers:")) || sscanf(ptr + 8, "%ld", &buffers) != 1)
+-                DEBUG("system statistic error -- cannot get real memory buffers amount\n");
+-        if (! (ptr = strstr(buf, "Cached:")) || sscanf(ptr + 7, "%ld", &cached) != 1)
+-                DEBUG("system statistic error -- cannot get real memory cache amount\n");
+-        if (! (ptr = strstr(buf, "SReclaimable:")) || sscanf(ptr + 13, "%ld", &slabreclaimable) != 1)
+-                DEBUG("system statistic error -- cannot get slab reclaimable memory amount\n");
+-        si->total_mem = systeminfo.mem_max - (uint64_t)(mem_free + buffers + cached + slabreclaimable) * 1024;
+ 
+         /* Swap */
+         if (! (ptr = strstr(buf, "SwapTotal:")) || sscanf(ptr + 10, "%ld", &swap_total) != 1) {
+-- 
+2.17.1
+

src/monit/patch/series

@@ -0,0 +1,2 @@
+# This series applies on GIT commit dc9bc1c949125140d967edfc598dfad47eedc552
+0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch