[Services] Restart BGP service upon unexpected critical process exit. (#4207)

dockers/docker-database/supervisord.conf.j2

@@ -9,7 +9,6 @@ events=PROCESS_STATE_EXITED
 autostart=true
 autorestart=unexpected
 
-
 [program:rsyslogd]
 command=/bin/bash -c "rm -f /var/run/rsyslogd.pid && /usr/sbin/rsyslogd -n"
 priority=1

dockers/docker-fpm-frr/Dockerfile.j2

@@ -50,6 +50,8 @@ COPY ["snmp.conf", "/etc/snmp/frr.conf"]
 COPY ["TSA", "/usr/bin/TSA"]
 COPY ["TSB", "/usr/bin/TSB"]
 COPY ["TSC", "/usr/bin/TSC"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSB && \
     chmod a+x /usr/bin/TSC

dockers/docker-fpm-frr/critical_processes

@@ -0,0 +1,5 @@
+zebra
+staticd
+bgpd
+fpmsyncd
+bgpcfgd

dockers/docker-fpm-frr/supervisord.conf

@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -25,7 +31,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=3
 autostart=false
-autorestart=false
+autorestart=unexpected
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog

dockers/docker-fpm-gobgp/Dockerfile.j2

@@ -24,5 +24,7 @@ COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
 COPY ["daemons", "/etc/quagga/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-fpm-gobgp/critical_processes

@@ -0,0 +1,2 @@
+gobgpd
+fpmsyncd

dockers/docker-fpm-gobgp/supervisord.conf

@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -15,7 +21,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=2
 autostart=false
-autorestart=false
+autorestart=unexpected
 stdout_logfile=syslog
 stderr_logfile=syslog
 

dockers/docker-fpm-quagga/Dockerfile.j2

@@ -33,5 +33,7 @@ RUN rm -rf /debs ~/.cache
 COPY ["bgpcfgd", "start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-fpm-quagga/critical_processes

@@ -0,0 +1,4 @@
+zebra
+bgpd
+fpmsyncd
+bgpcfgd

dockers/docker-fpm-quagga/supervisord.conf

@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -25,7 +31,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=3
 autostart=false
-autorestart=false
+autorestart=unexpected
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog

files/build_templates/single_instance/bgp.service.j2

@@ -3,12 +3,16 @@ Description=BGP container
 Requires=updategraph.service
 After=updategraph.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User={{ sonicadmin_user }}
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target

rules/docker-fpm-frr.mk

@@ -26,6 +26,8 @@ $(DOCKER_FPM_FRR)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_FRR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_FPM_FRR)_RUN_OPT += -v /etc/sonic/frr:/etc/frr:rw
 
+$(DOCKER_FPM_FRR)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += vtysh:/usr/bin/vtysh
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += TSA:/usr/bin/TSA
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += TSB:/usr/bin/TSB

rules/docker-fpm-gobgp.mk

@@ -9,3 +9,4 @@ SONIC_DOCKER_IMAGES += $(DOCKER_FPM_GOBGP)
 $(DOCKER_FPM_GOBGP)_CONTAINER_NAME = bgp
 $(DOCKER_FPM_GOBGP)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_GOBGP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_FPM_GOBPG)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

rules/docker-fpm-quagga.mk

@@ -10,4 +10,6 @@ $(DOCKER_FPM_QUAGGA)_CONTAINER_NAME = bgp
 $(DOCKER_FPM_QUAGGA)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_QUAGGA)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
+$(DOCKER_FPM_QUAGGA)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+
 $(DOCKER_FPM_QUAGGA)_BASE_IMAGE_FILES += vtysh:/usr/bin/vtysh