From 6eca27e5647f5af9adc149ba3858955ca679f803 Mon Sep 17 00:00:00 2001
From: Joe LeVeque <jleveque@users.noreply.github.com>
Date: Wed, 1 May 2019 08:02:38 -0700
Subject: [PATCH] [services] Restart SwSS service upon unexpected critical
 process exit (#2845)

* [service] Restart SwSS Docker container if orchagent exits unexpectedly

* Configure systemd to stop restarting swss if it attempts to restart more than 3 times in 20 minutes

* Move supervisor-proc-exit-listener script

* [docker-dhcp-relay] Enhance wait_for_intf.sh.j2 to utilize STATEDB

* Ensure dependent services stop/start/restart with SwSS

* Change 'StartLimitInterval' to 'StartLimitIntervalSec', as Stretch installs systemd 232 (>= v230)

* Also update journald.conf options

* Remove 'PartOf' option from unit files

* Add '$(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)' to new shared docker-orchagent makefile

* Make supervisor-proc-exit-listener script read from 'critical_processes' file inside container

* Update critical_processes file for swss container
---
 dockers/docker-dhcp-relay/wait_for_intf.sh.j2 | 38 +++++++-------
 dockers/docker-orchagent/Dockerfile.j2        |  2 +
 dockers/docker-orchagent/critical_processes   | 11 ++++
 dockers/docker-orchagent/supervisord.conf     |  8 ++-
 files/build_templates/dhcp_relay.service.j2   |  2 +-
 files/build_templates/radv.service.j2         |  2 +-
 files/build_templates/snmp.service.j2         |  3 ++
 files/build_templates/swss.service.j2         |  4 ++
 files/build_templates/teamd.service.j2        |  4 +-
 files/image_config/systemd/journald.conf      |  2 +-
 files/scripts/supervisor-proc-exit-listener   | 45 +++++++++++++++++
 rules/docker-dhcp-relay.mk                    |  2 +-
 rules/docker-orchagent.mk                     |  2 +-
 rules/scripts.mk                              |  6 ++-
 .../tests/sample_output/wait_for_intf.sh      | 50 +++++++++----------
 15 files changed, 126 insertions(+), 55 deletions(-)
 create mode 100644 dockers/docker-orchagent/critical_processes
 create mode 100755 files/scripts/supervisor-proc-exit-listener

diff --git a/dockers/docker-dhcp-relay/wait_for_intf.sh.j2 b/dockers/docker-dhcp-relay/wait_for_intf.sh.j2
index 037dc66ead..23133706cb 100644
--- a/dockers/docker-dhcp-relay/wait_for_intf.sh.j2
+++ b/dockers/docker-dhcp-relay/wait_for_intf.sh.j2
@@ -1,42 +1,40 @@
 #!/usr/bin/env bash
 
+STATE_DB_IDX="6"
+
+PORT_TABLE_PREFIX="PORT_TABLE"
+VLAN_TABLE_PREFIX="VLAN_TABLE"
+LAG_TABLE_PREFIX="LAG_TABLE"
+
 function wait_until_iface_ready
 {
-    IFACE=$1
+    TABLE_PREFIX=$1
+    IFACE=$2
 
-    echo "Waiting until interface $IFACE is up..."
+    echo "Waiting until interface $IFACE is ready..."
 
-    # Wait for the interface to come up (i.e., 'ip link show' returns 0)
-    until ip link show dev $IFACE up > /dev/null 2>&1; do
-        sleep 1
-    done
-
-    echo "Interface $IFACE is up"
-
-    echo "Waiting until interface $IFACE has an IPv4 address..."
-
-    # Wait until the interface gets assigned an IPv4 address
+    # Wait for the interface to come up
+    # (i.e., interface is present in STATE_DB and state is "ok")
     while true; do
-        IP=$(ip -4 addr show dev $IFACE | grep "inet " | awk '{ print $2 }' | cut -d '/' -f1)
-
-        if [ -n "$IP" ]; then
+        RESULT=$(redis-cli -n ${STATE_DB_IDX} HGET "${TABLE_PREFIX}|${IFACE}" "state" 2> /dev/null)
+        if [ x"$RESULT" == x"ok" ]; then
             break
         fi
 
         sleep 1
     done
 
-    echo "Interface $IFACE is configured with IP $IP"
+    echo "Interface ${IFACE} is ready!"
 }
 
 
-# Wait for all interfaces to come up and have IPv4 addresses assigned
+# Wait for all interfaces to be up and ready
 {% for (name, prefix) in INTERFACE %}
-wait_until_iface_ready {{ name }}
+wait_until_iface_ready ${PORT_TABLE_PREFIX} {{ name }}
 {% endfor %}
 {% for (name, prefix) in VLAN_INTERFACE %}
-wait_until_iface_ready {{ name }}
+wait_until_iface_ready ${VLAN_TABLE_PREFIX} {{ name }}
 {% endfor %}
 {% for (name, prefix) in PORTCHANNEL_INTERFACE %}
-wait_until_iface_ready {{ name }}
+wait_until_iface_ready ${LAG_TABLE_PREFIX} {{ name }}
 {% endfor %}
diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2
index d29ea6a890..8f480a5328 100755
--- a/dockers/docker-orchagent/Dockerfile.j2
+++ b/dockers/docker-orchagent/Dockerfile.j2
@@ -41,6 +41,8 @@ COPY ["files/arp_update", "/usr/bin"]
 COPY ["enable_counters.py", "/usr/bin"]
 COPY ["start.sh", "orchagent.sh", "swssconfig.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor/"]
 
 ## Copy all Jinja2 template files into the templates folder
 COPY ["*.j2", "/usr/share/sonic/templates/"]
diff --git a/dockers/docker-orchagent/critical_processes b/dockers/docker-orchagent/critical_processes
new file mode 100644
index 0000000000..c0f441ecc0
--- /dev/null
+++ b/dockers/docker-orchagent/critical_processes
@@ -0,0 +1,11 @@
+orchagent
+portsyncd
+neighsyncd
+vlanmgrd
+intfmgrd
+portmgrd
+buffermgrd
+vrfmgrd
+nbrmgrd
+vxlanmgrd
+intfsyncd
diff --git a/dockers/docker-orchagent/supervisord.conf b/dockers/docker-orchagent/supervisord.conf
index 4479a27f2a..46b37e77c0 100644
--- a/dockers/docker-orchagent/supervisord.conf
+++ b/dockers/docker-orchagent/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -15,7 +21,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=2
 autostart=false
-autorestart=false
+autorestart=unexpected
 stdout_logfile=syslog
 stderr_logfile=syslog
 
diff --git a/files/build_templates/dhcp_relay.service.j2 b/files/build_templates/dhcp_relay.service.j2
index 5a462e5397..7ec133c87a 100644
--- a/files/build_templates/dhcp_relay.service.j2
+++ b/files/build_templates/dhcp_relay.service.j2
@@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{ docker_container_name }}.sh wait
 ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
 
 [Install]
-WantedBy=multi-user.target teamd.service
+WantedBy=multi-user.target swss.service teamd.service
diff --git a/files/build_templates/radv.service.j2 b/files/build_templates/radv.service.j2
index 8cda2fdd0a..4f1c67e661 100644
--- a/files/build_templates/radv.service.j2
+++ b/files/build_templates/radv.service.j2
@@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{ docker_container_name }}.sh wait
 ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target swss.service
diff --git a/files/build_templates/snmp.service.j2 b/files/build_templates/snmp.service.j2
index 416156d5a8..43f46bd2b9 100644
--- a/files/build_templates/snmp.service.j2
+++ b/files/build_templates/snmp.service.j2
@@ -9,3 +9,6 @@ Before=ntp-config.service
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+
+[Install]
+WantedBy=multi-user.target swss.service
diff --git a/files/build_templates/swss.service.j2 b/files/build_templates/swss.service.j2
index b391f95de5..048c9c3470 100644
--- a/files/build_templates/swss.service.j2
+++ b/files/build_templates/swss.service.j2
@@ -9,6 +9,8 @@ Requires=nps-modules-4.9.0-8-2-amd64.service
 After=database.service updategraph.service
 After=interfaces-config.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User=root
@@ -16,6 +18,8 @@ Environment=sonic_asic_platform={{ sonic_asic_platform }}
 ExecStartPre=/usr/local/bin/swss.sh start
 ExecStart=/usr/local/bin/swss.sh wait
 ExecStop=/usr/local/bin/swss.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/files/build_templates/teamd.service.j2 b/files/build_templates/teamd.service.j2
index 8034698ecc..58c858effb 100644
--- a/files/build_templates/teamd.service.j2
+++ b/files/build_templates/teamd.service.j2
@@ -1,6 +1,6 @@
 [Unit]
 Description=TEAMD container
-Requires=updategraph.service
+Requires=updategraph.service swss.service
 After=updategraph.service swss.service
 Before=ntp-config.service
 
@@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target swss.service
diff --git a/files/image_config/systemd/journald.conf b/files/image_config/systemd/journald.conf
index 0f3d1b5ec2..214bd099a2 100644
--- a/files/image_config/systemd/journald.conf
+++ b/files/image_config/systemd/journald.conf
@@ -13,7 +13,7 @@
 #Seal=yes
 #SplitMode=uid
 #SyncIntervalSec=5m
-#RateLimitInterval=30s
+#RateLimitIntervalSec=30s
 #RateLimitBurst=1000
 SystemMaxUse=50M
 #SystemKeepFree=
diff --git a/files/scripts/supervisor-proc-exit-listener b/files/scripts/supervisor-proc-exit-listener
new file mode 100755
index 0000000000..6bc62fc400
--- /dev/null
+++ b/files/scripts/supervisor-proc-exit-listener
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+import os
+import signal
+import sys
+import syslog
+
+from supervisor import childutils
+
+# Contents of file should be the names of critical processes (as defined in
+# supervisor.conf file), one per line
+CRITICAL_PROCESSES_FILE = '/etc/supervisor/critical_processes'
+
+def main():
+    # Read the list of critical processes from a file
+    with open(CRITICAL_PROCESSES_FILE, 'r') as f:
+        critical_processes = [line.rstrip('\n') for line in f]
+
+    while True:
+        # Transition from ACKNOWLEDGED to READY
+        childutils.listener.ready()
+
+        line = sys.stdin.readline()
+        headers = childutils.get_headers(line)
+        payload = sys.stdin.read(int(headers['len']))
+
+        # Transition from READY to ACKNOWLEDGED
+        childutils.listener.ok()
+
+        # We only care about PROCESS_STATE_EXITED events
+        if headers['eventname'] == 'PROCESS_STATE_EXITED':
+            payload_headers, payload_data = childutils.eventdata(payload + '\n')
+
+            expected = int(payload_headers['expected'])
+            processname = payload_headers['processname']
+
+            # If a critical process exited unexpectedly, terminate supervisor
+            if expected == 0 and processname in critical_processes:
+                MSG_FORMAT_STR = "Process {} exited unxepectedly. Terminating supervisor..."
+                msg = MSG_FORMAT_STR.format(payload_headers['processname'])
+                syslog.syslog(syslog.LOG_INFO, msg)
+                os.kill(os.getppid(), signal.SIGTERM)
+
+if __name__ == "__main__":
+    main()
diff --git a/rules/docker-dhcp-relay.mk b/rules/docker-dhcp-relay.mk
index f1f19d974a..d15bdbf84d 100644
--- a/rules/docker-dhcp-relay.mk
+++ b/rules/docker-dhcp-relay.mk
@@ -6,7 +6,7 @@ DOCKER_DHCP_RELAY_DBG = $(DOCKER_DHCP_RELAY_STEM)-$(DBG_IMAGE_MARK).gz
 
 $(DOCKER_DHCP_RELAY)_PATH = $(DOCKERS_PATH)/$(DOCKER_DHCP_RELAY_STEM)
 
-$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_COMMON) $(ISC_DHCP_RELAY)
+$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_COMMON) $(ISC_DHCP_RELAY) $(REDIS_TOOLS)
 $(DOCKER_DHCP_RELAY)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS)
 $(DOCKER_DHCP_RELAY)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)
 
diff --git a/rules/docker-orchagent.mk b/rules/docker-orchagent.mk
index ea63730817..20536adc25 100644
--- a/rules/docker-orchagent.mk
+++ b/rules/docker-orchagent.mk
@@ -34,4 +34,4 @@ $(DOCKER_ORCHAGENT)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_ORCHAGENT)_RUN_OPT += -v /var/log/swss:/var/log/swss:rw
 
 $(DOCKER_ORCHAGENT)_BASE_IMAGE_FILES += swssloglevel:/usr/bin/swssloglevel
-$(DOCKER_ORCHAGENT)_FILES += $(ARP_UPDATE_SCRIPT)
+$(DOCKER_ORCHAGENT)_FILES += $(ARP_UPDATE_SCRIPT) $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
diff --git a/rules/scripts.mk b/rules/scripts.mk
index 6010b10b83..8c6d0324fc 100644
--- a/rules/scripts.mk
+++ b/rules/scripts.mk
@@ -11,9 +11,13 @@ $(BUFFERS_CONFIG_TEMPLATE)_PATH = files/build_templates
 QOS_CONFIG_TEMPLATE = qos_config.j2
 $(QOS_CONFIG_TEMPLATE)_PATH = files/build_templates
 
+SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT = supervisor-proc-exit-listener
+$(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)_PATH = files/scripts
+
 SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
                     $(ARP_UPDATE_SCRIPT) \
                     $(BUFFERS_CONFIG_TEMPLATE) \
-                    $(QOS_CONFIG_TEMPLATE)
+                    $(QOS_CONFIG_TEMPLATE) \
+                    $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
 
 
diff --git a/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh b/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh
index 0f06235eb0..383f7cb389 100644
--- a/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh
+++ b/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh
@@ -1,43 +1,41 @@
 #!/usr/bin/env bash
 
+STATE_DB_IDX="6"
+
+PORT_TABLE_PREFIX="PORT_TABLE"
+VLAN_TABLE_PREFIX="VLAN_TABLE"
+LAG_TABLE_PREFIX="LAG_TABLE"
+
 function wait_until_iface_ready
 {
-    IFACE=$1
+    TABLE_PREFIX=$1
+    IFACE=$2
 
-    echo "Waiting until interface $IFACE is up..."
+    echo "Waiting until interface $IFACE is ready..."
 
-    # Wait for the interface to come up (i.e., 'ip link show' returns 0)
-    until ip link show dev $IFACE up > /dev/null 2>&1; do
-        sleep 1
-    done
-
-    echo "Interface $IFACE is up"
-
-    echo "Waiting until interface $IFACE has an IPv4 address..."
-
-    # Wait until the interface gets assigned an IPv4 address
+    # Wait for the interface to come up
+    # (i.e., interface is present in STATE_DB and state is "ok")
     while true; do
-        IP=$(ip -4 addr show dev $IFACE | grep "inet " | awk '{ print $2 }' | cut -d '/' -f1)
-
-        if [ -n "$IP" ]; then
+        RESULT=$(redis-cli -n ${STATE_DB_IDX} HGET "${TABLE_PREFIX}|${IFACE}" "state" 2> /dev/null)
+        if [ x"$RESULT" == x"ok" ]; then
             break
         fi
 
         sleep 1
     done
 
-    echo "Interface $IFACE is configured with IP $IP"
+    echo "Interface ${IFACE} is ready!"
 }
 
 
-# Wait for all interfaces to come up and have IPv4 addresses assigned
-wait_until_iface_ready Vlan1000
-wait_until_iface_ready PortChannel01
-wait_until_iface_ready PortChannel01
-wait_until_iface_ready PortChannel02
-wait_until_iface_ready PortChannel02
-wait_until_iface_ready PortChannel03
-wait_until_iface_ready PortChannel03
-wait_until_iface_ready PortChannel04
-wait_until_iface_ready PortChannel04
+# Wait for all interfaces to be up and ready
+wait_until_iface_ready ${VLAN_TABLE_PREFIX} Vlan1000
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel01
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel01
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel02
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel02
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel03
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel03
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel04
+wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel04