[Services] Restart SNMP service upon unexpected critical process exit. (#3650)

Signed-off-by: Yong Zhao <yozhao@microsoft.com>

dockers/docker-snmp-sv2/Dockerfile.j2

@@ -84,6 +84,8 @@ RUN apt-get -y purge     \
 COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 # Although exposing ports is not needed for host net mode, keep it for possible bridge mode
 EXPOSE 161/udp 162/udp

dockers/docker-snmp-sv2/critical_processes

@@ -0,0 +1,2 @@
+snmpd
+snmp-subagent

dockers/docker-snmp-sv2/supervisord.conf

@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -15,7 +21,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=2
 autostart=false
-autorestart=false
+autorestart=unexpected
 stdout_logfile=syslog
 stderr_logfile=syslog
 

files/build_templates/snmp.service.j2

@@ -4,11 +4,15 @@ Requires=updategraph.service
 Requisite=swss.service
 After=updategraph.service swss.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target swss.service

rules/docker-snmp-sv2.mk

@@ -30,3 +30,4 @@ $(DOCKER_SNMP_SV2)_RUN_OPT += --net=host --privileged -t
 $(DOCKER_SNMP_SV2)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 # mount Arista platform python libraries to support corresponding platforms SNMP power status query
 $(DOCKER_SNMP_SV2)_RUN_OPT += -v /usr/lib/python3/dist-packages/arista:/usr/lib/python3/dist-packages/arista:ro
+$(DOCKER_SNMP_SV2)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)