From 5e07b252ffedd98ed57199fba093c80fe25472a9 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 30 Dec 2019 18:25:57 -0800 Subject: [PATCH] [monit] Build from source and patch to use MemAvailable value if available on system (#3875) --- .gitignore | 3 + build_debian.sh | 5 -- .../build_templates/sonic_debian_extension.j2 | 8 +++ files/image_config/monit/conf.d/sonic-host | 22 ++++++ files/image_config/monit/monitrc | 19 +---- rules/monit.mk | 14 ++++ slave.mk | 3 +- src/monit/Makefile | 33 +++++++++ ...ry_sysdep-Use-MemAvailable-value-if-.patch | 70 +++++++++++++++++++ src/monit/patch/series | 2 + 10 files changed, 157 insertions(+), 22 deletions(-) create mode 100644 files/image_config/monit/conf.d/sonic-host create mode 100644 rules/monit.mk create mode 100644 src/monit/Makefile create mode 100644 src/monit/patch/0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch create mode 100644 src/monit/patch/series diff --git a/.gitignore b/.gitignore index 7394541687..1f921971d8 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,9 @@ src/lldpd/* !src/lldpd/patch/ src/lm-sensors/* !src/lm-sensors/Makefile +src/monit/* +!src/monit/Makefile +!src/monit/patch/ src/mpdecimal/* !src/mpdecimal/Makefile src/python-click/* diff --git a/build_debian.sh b/build_debian.sh index fb3ecfef6c..eba6f59ef0 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -240,7 +240,6 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in openssh-server \ python \ python-setuptools \ - monit \ python-apt \ traceroute \ iputils-ping \ @@ -348,10 +347,6 @@ EOF sudo sed -i 's/^ListenAddress ::/#ListenAddress ::/' $FILESYSTEM_ROOT/etc/ssh/sshd_config sudo sed -i 's/^#ListenAddress 0.0.0.0/ListenAddress 0.0.0.0/' $FILESYSTEM_ROOT/etc/ssh/sshd_config -## Config monit -sudo cp files/image_config/monit/monitrc $FILESYSTEM_ROOT/etc/monit/ -sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/monitrc - ## Config sysctl sudo mkdir -p $FILESYSTEM_ROOT/var/core sudo augtool --autosave " diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 068ea68b84..0e6d07fb00 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -166,6 +166,14 @@ sudo sed -i -e '/^passwd/s/ tacplus//' $FILESYSTEM_ROOT/etc/nsswitch.conf sudo DEBIAN_FRONTEND=noninteractive dpkg --root=$FILESYSTEM_ROOT -i $debs_path/kdump-tools_*.deb || \ sudo LANG=C DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=truechroot $FILESYSTEM_ROOT apt-get -q --no-install-suggests --no-install-recommends --force-no install +# Install custom-built monit package and SONiC configuration files +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/monit_*.deb || \ + sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f +sudo cp $IMAGE_CONFIGS/monit/monitrc $FILESYSTEM_ROOT/etc/monit/ +sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/monitrc +sudo cp $IMAGE_CONFIGS/monit/conf.d/* $FILESYSTEM_ROOT/etc/monit/conf.d/ +sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/conf.d/* + # Copy crontabs sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/ diff --git a/files/image_config/monit/conf.d/sonic-host b/files/image_config/monit/conf.d/sonic-host new file mode 100644 index 0000000000..8eaa1671d8 --- /dev/null +++ b/files/image_config/monit/conf.d/sonic-host @@ -0,0 +1,22 @@ +############################################################################### +## Monit configuration for SONiC host OS +## +## This includes system-level monitoring as well as processes which +## run in the host OS (i.e., not inside a Docker container) +############################################################################### + +check filesystem root-overlay with path / + if space usage > 90% for 5 times within 10 cycles then alert + +check filesystem var-log with path /var/log + if space usage > 90% for 5 times within 10 cycles then alert + +check system $HOST + if memory usage > 90% for 5 times within 10 cycles then alert + if cpu usage (user) > 90% for 5 times within 10 cycles then alert + if cpu usage (system) > 90% for 5 times within 10 cycles then alert + +check process rsyslog with pidfile /var/run/rsyslogd.pid + start program = "/bin/systemctl start rsyslog.service" + stop program = "/bin/systemctl stop rsyslog.service" + if totalmem > 800 MB for 5 times within 10 cycles then restart diff --git a/files/image_config/monit/monitrc b/files/image_config/monit/monitrc index f9350733fa..7864069e3a 100644 --- a/files/image_config/monit/monitrc +++ b/files/image_config/monit/monitrc @@ -24,8 +24,7 @@ ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # -# set logfile /var/log/monit.log -set logfile syslog + set logfile syslog # # ## Set the location of the Monit lock file which stores the process id of the @@ -153,8 +152,8 @@ set logfile syslog ## commands to a running Monit daemon. See the Monit Wiki if you want to ## enable SSL for the HTTP interface. # -set httpd unixsocket /var/run/monit.sock and - allow localhost # allow localhost to connect to the server and + set httpd unixsocket /var/run/monit.sock and + allow localhost # allow localhost to connect to the server # ############################################################################### ## Services @@ -294,15 +293,3 @@ set httpd unixsocket /var/run/monit.sock and include /etc/monit/conf.d/* include /etc/monit/conf-enabled/* # -check filesystem root-overlay with path / - if space usage > 90% for 5 times within 10 cycles then alert -check filesystem var-log with path /var/log - if space usage > 90% for 5 times within 10 cycles then alert -check system $HOST - if memory usage > 90% for 5 times within 10 cycles then alert - if cpu usage (user) > 90% for 5 times within 10 cycles then alert - if cpu usage (system) > 90% for 5 times within 10 cycles then alert -check process rsyslog with pidfile /var/run/rsyslogd.pid - start program = "/bin/systemctl start rsyslog.service" - stop program = "/bin/systemctl stop rsyslog.service" - if totalmem > 800 MB for 5 times within 10 cycles then restart diff --git a/rules/monit.mk b/rules/monit.mk new file mode 100644 index 0000000000..d4c73453e4 --- /dev/null +++ b/rules/monit.mk @@ -0,0 +1,14 @@ +# monit package + +MONIT_VERSION = 5.20.0-6 + +export MONIT_VERSION + +MONIT = monit_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb +$(MONIT)_SRC_PATH = $(SRC_PATH)/monit +SONIC_MAKE_DEBS += $(MONIT) + +SONIC_STRETCH_DEBS += $(MONIT) + +MONIT_DBG = monit-dbgsym_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb +$(eval $(call add_derived_package,$(MONIT),$(MONIT_DBG))) diff --git a/slave.mk b/slave.mk index 0d0243c188..147b2a95a7 100644 --- a/slave.mk +++ b/slave.mk @@ -616,7 +616,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(IFUPDOWN2) \ $(KDUMP_TOOLS) \ $(LIBPAM_TACPLUS) \ - $(LIBNSS_TACPLUS)) \ + $(LIBNSS_TACPLUS) \ + $(MONIT)) \ $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \ $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \ $(addprefix $(STRETCH_FILES_PATH)/, $(if $(filter $(CONFIGURED_ARCH),amd64), $(IXGBE_DRIVER))) \ diff --git a/src/monit/Makefile b/src/monit/Makefile new file mode 100644 index 0000000000..4ad9edd791 --- /dev/null +++ b/src/monit/Makefile @@ -0,0 +1,33 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +MAIN_TARGET = monit_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb +DERIVED_TARGETS = monit-dbgsym_$(MONIT_VERSION)_$(CONFIGURED_ARCH).deb + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + # Remove any stale files + rm -rf ./monit + + # Clone monit repo + git clone https://salsa.debian.org/sk-guest/monit.git + pushd ./monit + + # Reset HEAD to the commit of the proper tag + # NOTE: Using "git checkout " here detaches our HEAD, + # which stg doesn't like, so we use this method instead + # NOTE: For some reason, tags in the Debian monit repo are prefixed with "1%" + git reset --hard debian/1\%$(MONIT_VERSION) + + # Apply patches + stg init + stg import -s ../patch/series + + # Build source and Debian packages + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + popd + + # Move the newly-built .deb packages to the destination directory + mv $(DERIVED_TARGETS) $* $(DEST)/ + +$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) diff --git a/src/monit/patch/0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch b/src/monit/patch/0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch new file mode 100644 index 0000000000..9c67d153cb --- /dev/null +++ b/src/monit/patch/0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch @@ -0,0 +1,70 @@ +From c392362c9c1d57256b7e8ab7c77926824677fd73 Mon Sep 17 00:00:00 2001 +From: Joe LeVeque +Date: Tue, 19 Nov 2019 01:51:13 +0000 +Subject: [PATCH] [used_system_memory_sysdep] Use 'MemAvailable' value if + available + +--- + src/process/sysdep_LINUX.c | 35 +++++++++++++++++++++++------------ + 1 file changed, 23 insertions(+), 12 deletions(-) + +diff --git a/src/process/sysdep_LINUX.c b/src/process/sysdep_LINUX.c +index 0d18f85..221e785 100644 +--- a/src/process/sysdep_LINUX.c ++++ b/src/process/sysdep_LINUX.c +@@ -335,6 +335,7 @@ int getloadavg_sysdep(double *loadv, int nelem) { + boolean_t used_system_memory_sysdep(SystemInfo_T *si) { + char *ptr; + char buf[2048]; ++ unsigned long mem_available = 0UL; + unsigned long mem_free = 0UL; + unsigned long buffers = 0UL; + unsigned long cached = 0UL; +@@ -343,22 +344,32 @@ boolean_t used_system_memory_sysdep(SystemInfo_T *si) { + unsigned long swap_free = 0UL; + + if (! file_readProc(buf, sizeof(buf), "meminfo", -1, NULL)) { +- LogError("system statistic error -- cannot get real memory free amount\n"); ++ LogError("system statistic error -- cannot read /proc/meminfo\n"); + goto error; + } + +- /* Memory */ +- if (! (ptr = strstr(buf, "MemFree:")) || sscanf(ptr + 8, "%ld", &mem_free) != 1) { +- LogError("system statistic error -- cannot get real memory free amount\n"); +- goto error; ++ /* ++ * Memory ++ * ++ * First, check if the "MemAvailable" value is available on this system. If it is, we will ++ * use it. Otherwise we will attempt to calculate the amount of available memory ourself. ++ */ ++ if ((ptr = strstr(buf, "MemAvailable:")) && sscanf(ptr + 13, "%ld", &mem_available) == 1) { ++ si->total_mem = systeminfo.mem_max - (uint64_t)mem_available * 1024; ++ } else { ++ DEBUG("'MemAvailable' value not available on this system. Attempting to calculate available memory manually...\n"); ++ if (! (ptr = strstr(buf, "MemFree:")) || sscanf(ptr + 8, "%ld", &mem_free) != 1) { ++ LogError("system statistic error -- cannot get real memory free amount\n"); ++ goto error; ++ } ++ if (! (ptr = strstr(buf, "Buffers:")) || sscanf(ptr + 8, "%ld", &buffers) != 1) ++ DEBUG("system statistic error -- cannot get real memory buffers amount\n"); ++ if (! (ptr = strstr(buf, "Cached:")) || sscanf(ptr + 7, "%ld", &cached) != 1) ++ DEBUG("system statistic error -- cannot get real memory cache amount\n"); ++ if (! (ptr = strstr(buf, "SReclaimable:")) || sscanf(ptr + 13, "%ld", &slabreclaimable) != 1) ++ DEBUG("system statistic error -- cannot get slab reclaimable memory amount\n"); ++ si->total_mem = systeminfo.mem_max - (uint64_t)(mem_free + buffers + cached + slabreclaimable) * 1024; + } +- if (! (ptr = strstr(buf, "Buffers:")) || sscanf(ptr + 8, "%ld", &buffers) != 1) +- DEBUG("system statistic error -- cannot get real memory buffers amount\n"); +- if (! (ptr = strstr(buf, "Cached:")) || sscanf(ptr + 7, "%ld", &cached) != 1) +- DEBUG("system statistic error -- cannot get real memory cache amount\n"); +- if (! (ptr = strstr(buf, "SReclaimable:")) || sscanf(ptr + 13, "%ld", &slabreclaimable) != 1) +- DEBUG("system statistic error -- cannot get slab reclaimable memory amount\n"); +- si->total_mem = systeminfo.mem_max - (uint64_t)(mem_free + buffers + cached + slabreclaimable) * 1024; + + /* Swap */ + if (! (ptr = strstr(buf, "SwapTotal:")) || sscanf(ptr + 10, "%ld", &swap_total) != 1) { +-- +2.17.1 + diff --git a/src/monit/patch/series b/src/monit/patch/series new file mode 100644 index 0000000000..15fcdd50c8 --- /dev/null +++ b/src/monit/patch/series @@ -0,0 +1,2 @@ +# This series applies on GIT commit dc9bc1c949125140d967edfc598dfad47eedc552 +0001-used_system_memory_sysdep-Use-MemAvailable-value-if-.patch