matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[services] Restart SwSS service upon unexpected critical process exit (#2845) (#2852)

Browse Source
This commit is contained in:
Joe LeVeque 2019-07-29 18:10:26 -07:00 committed by GitHub
parent 7cf90ec441
commit 29bbd86862
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 124 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dockers/docker-dhcp-relay/wait_for_intf.sh.j2
View File

@ -1,42 +1,40 @@
#!/usr/bin/env bash
STATE_DB_IDX="6"
PORT_TABLE_PREFIX="PORT_TABLE"
VLAN_TABLE_PREFIX="VLAN_TABLE"
LAG_TABLE_PREFIX="LAG_TABLE"
function wait_until_iface_ready
{
IFACE=$1
TABLE_PREFIX=$1
IFACE=$2
echo "Waiting until interface $IFACE is up..."
echo "Waiting until interface $IFACE is ready..."
# Wait for the interface to come up (i.e., 'ip link show' returns 0)
until ip link show dev $IFACE up > /dev/null 2>&1; do
sleep 1
done
echo "Interface $IFACE is up"
echo "Waiting until interface $IFACE has an IPv4 address..."
# Wait until the interface gets assigned an IPv4 address
# Wait for the interface to come up
# (i.e., interface is present in STATE_DB and state is "ok")
while true; do
IP=$(ip -4 addr show dev $IFACE | grep "inet " | awk '{ print $2 }' | cut -d '/' -f1)
if [ -n "$IP" ]; then
RESULT=$(redis-cli -n ${STATE_DB_IDX} HGET "${TABLE_PREFIX}|${IFACE}" "state" 2> /dev/null)
if [ x"$RESULT" == x"ok" ]; then
break
fi
sleep 1
done
echo "Interface $IFACE is configured with IP $IP"
echo "Interface ${IFACE} is ready!"
}
# Wait for all interfaces to come up and have IPv4 addresses assigned
# Wait for all interfaces to be up and ready
{% for (name, prefix) in INTERFACE %}
wait_until_iface_ready {{ name }}
wait_until_iface_ready ${PORT_TABLE_PREFIX} {{ name }}
{% endfor %}
{% for (name, prefix) in VLAN_INTERFACE %}
wait_until_iface_ready {{ name }}
wait_until_iface_ready ${VLAN_TABLE_PREFIX} {{ name }}
{% endfor %}
{% for (name, prefix) in PORTCHANNEL_INTERFACE %}
wait_until_iface_ready {{ name }}
wait_until_iface_ready ${LAG_TABLE_PREFIX} {{ name }}
{% endfor %}

2
dockers/docker-orchagent/Dockerfile.j2
View File

@ -40,6 +40,8 @@ COPY ["files/arp_update", "/usr/bin"]
COPY ["enable_counters.py", "/usr/bin"]
COPY ["start.sh", "orchagent.sh", "swssconfig.sh", "/usr/bin/"]
COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
COPY ["critical_processes", "/etc/supervisor/"]
## Copy all Jinja2 template files into the templates folder
COPY ["*.j2", "/usr/share/sonic/templates/"]

9
dockers/docker-orchagent/critical_processes Normal file
View File

@ -0,0 +1,9 @@
orchagent
portsyncd
neighsyncd
vlanmgrd
intfmgrd
portmgrd
buffermgrd
vrfmgrd
nbrmgrd

8
dockers/docker-orchagent/supervisord.conf
View File

@ -3,6 +3,12 @@ logfile_maxbytes=1MB
logfile_backups=2
nodaemon=true
[eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener
events=PROCESS_STATE_EXITED
autostart=true
autorestart=unexpected
[program:start.sh]
command=/usr/bin/start.sh
priority=1
@ -15,7 +21,7 @@ stderr_logfile=syslog
command=/usr/sbin/rsyslogd -n
priority=2
autostart=false
autorestart=false
autorestart=unexpected
stdout_logfile=syslog
stderr_logfile=syslog

2
files/build_templates/dhcp_relay.service.j2
View File

@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{ docker_container_name }}.sh wait
ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
[Install]
WantedBy=multi-user.target teamd.service
WantedBy=multi-user.target swss.service teamd.service

2
files/build_templates/radv.service.j2
View File

@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{ docker_container_name }}.sh wait
ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
[Install]
WantedBy=multi-user.target
WantedBy=multi-user.target swss.service

3
files/build_templates/snmp.service.j2
View File

@ -9,3 +9,6 @@ Before=ntp-config.service
ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
ExecStart=/usr/bin/{{docker_container_name}}.sh wait
ExecStop=/usr/bin/{{docker_container_name}}.sh stop
[Install]
WantedBy=multi-user.target swss.service

4
files/build_templates/swss.service.j2
View File

@ -9,6 +9,8 @@ Requires=nps-modules-4.9.0-9-amd64.service
After=database.service updategraph.service
After=interfaces-config.service
Before=ntp-config.service
StartLimitIntervalSec=1200
StartLimitBurst=3
[Service]
User=root
@ -16,6 +18,8 @@ Environment=sonic_asic_platform={{ sonic_asic_platform }}
ExecStartPre=/usr/local/bin/swss.sh start
ExecStart=/usr/local/bin/swss.sh wait
ExecStop=/usr/local/bin/swss.sh stop
Restart=always
RestartSec=30
[Install]
WantedBy=multi-user.target

4
files/build_templates/teamd.service.j2
View File

@ -1,6 +1,6 @@
[Unit]
Description=TEAMD container
Requires=updategraph.service
Requires=updategraph.service swss.service
After=updategraph.service swss.service
Before=ntp-config.service
@ -11,4 +11,4 @@ ExecStart=/usr/bin/{{docker_container_name}}.sh wait
ExecStop=/usr/bin/{{docker_container_name}}.sh stop
[Install]
WantedBy=multi-user.target
WantedBy=multi-user.target swss.service

2
files/image_config/systemd/journald.conf
View File

@ -13,7 +13,7 @@
#Seal=yes
#SplitMode=uid
#SyncIntervalSec=5m
#RateLimitInterval=30s
#RateLimitIntervalSec=30s
#RateLimitBurst=1000
SystemMaxUse=50M
#SystemKeepFree=

45
files/scripts/supervisor-proc-exit-listener Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env python
import os
import signal
import sys
import syslog
from supervisor import childutils
# Contents of file should be the names of critical processes (as defined in
# supervisor.conf file), one per line
CRITICAL_PROCESSES_FILE = '/etc/supervisor/critical_processes'
def main():
# Read the list of critical processes from a file
with open(CRITICAL_PROCESSES_FILE, 'r') as f:
critical_processes = [line.rstrip('\n') for line in f]
while True:
# Transition from ACKNOWLEDGED to READY
childutils.listener.ready()
line = sys.stdin.readline()
headers = childutils.get_headers(line)
payload = sys.stdin.read(int(headers['len']))
# Transition from READY to ACKNOWLEDGED
childutils.listener.ok()
# We only care about PROCESS_STATE_EXITED events
if headers['eventname'] == 'PROCESS_STATE_EXITED':
payload_headers, payload_data = childutils.eventdata(payload + '\n')
expected = int(payload_headers['expected'])
processname = payload_headers['processname']
# If a critical process exited unexpectedly, terminate supervisor
if expected == 0 and processname in critical_processes:
MSG_FORMAT_STR = "Process {} exited unxepectedly. Terminating supervisor..."
msg = MSG_FORMAT_STR.format(payload_headers['processname'])
syslog.syslog(syslog.LOG_INFO, msg)
os.kill(os.getppid(), signal.SIGTERM)
if __name__ == "__main__":
main()

2
platform/template/docker-orchagent-base.mk
View File

@ -32,4 +32,4 @@ $(DOCKER_ORCHAGENT_BASE)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
$(DOCKER_ORCHAGENT_BASE)_RUN_OPT += -v /var/log/swss:/var/log/swss:rw
$(DOCKER_ORCHAGENT_BASE)_BASE_IMAGE_FILES += swssloglevel:/usr/bin/swssloglevel
$(DOCKER_ORCHAGENT_BASE)_FILES += $(ARP_UPDATE_SCRIPT)
$(DOCKER_ORCHAGENT_BASE)_FILES += $(ARP_UPDATE_SCRIPT) $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

2
rules/docker-dhcp-relay.mk
View File

@ -6,7 +6,7 @@ DOCKER_DHCP_RELAY_DBG = $(DOCKER_DHCP_RELAY_STEM)-$(DBG_IMAGE_MARK).gz
$(DOCKER_DHCP_RELAY)_PATH = $(DOCKERS_PATH)/docker-dhcp-relay
$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_COMMON) $(ISC_DHCP_RELAY)
$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_COMMON) $(ISC_DHCP_RELAY) $(REDIS_TOOLS)
$(DOCKER_DHCP_RELAY)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE)_DBG_DEPENDS)
$(DOCKER_DHCP_RELAY)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE)_DBG_IMAGE_PACKAGES)

8
rules/scripts.mk
View File

@ -11,9 +11,11 @@ $(BUFFERS_CONFIG_TEMPLATE)_PATH = files/build_templates
QOS_CONFIG_TEMPLATE = qos_config.j2
$(QOS_CONFIG_TEMPLATE)_PATH = files/build_templates
SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT = supervisor-proc-exit-listener
$(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)_PATH = files/scripts
SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
$(ARP_UPDATE_SCRIPT) \
$(BUFFERS_CONFIG_TEMPLATE) \
$(QOS_CONFIG_TEMPLATE)
$(QOS_CONFIG_TEMPLATE) \
$(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

50
src/sonic-config-engine/tests/sample_output/wait_for_intf.sh
View File

@ -1,43 +1,41 @@
#!/usr/bin/env bash
STATE_DB_IDX="6"
PORT_TABLE_PREFIX="PORT_TABLE"
VLAN_TABLE_PREFIX="VLAN_TABLE"
LAG_TABLE_PREFIX="LAG_TABLE"
function wait_until_iface_ready
{
IFACE=$1
TABLE_PREFIX=$1
IFACE=$2
echo "Waiting until interface $IFACE is up..."
echo "Waiting until interface $IFACE is ready..."
# Wait for the interface to come up (i.e., 'ip link show' returns 0)
until ip link show dev $IFACE up > /dev/null 2>&1; do
sleep 1
done
echo "Interface $IFACE is up"
echo "Waiting until interface $IFACE has an IPv4 address..."
# Wait until the interface gets assigned an IPv4 address
# Wait for the interface to come up
# (i.e., interface is present in STATE_DB and state is "ok")
while true; do
IP=$(ip -4 addr show dev $IFACE | grep "inet " | awk '{ print $2 }' | cut -d '/' -f1)
if [ -n "$IP" ]; then
RESULT=$(redis-cli -n ${STATE_DB_IDX} HGET "${TABLE_PREFIX}|${IFACE}" "state" 2> /dev/null)
if [ x"$RESULT" == x"ok" ]; then
break
fi
sleep 1
done
echo "Interface $IFACE is configured with IP $IP"
echo "Interface ${IFACE} is ready!"
}
# Wait for all interfaces to come up and have IPv4 addresses assigned
wait_until_iface_ready Vlan1000
wait_until_iface_ready PortChannel01
wait_until_iface_ready PortChannel01
wait_until_iface_ready PortChannel02
wait_until_iface_ready PortChannel02
wait_until_iface_ready PortChannel03
wait_until_iface_ready PortChannel03
wait_until_iface_ready PortChannel04
wait_until_iface_ready PortChannel04
# Wait for all interfaces to be up and ready
wait_until_iface_ready ${VLAN_TABLE_PREFIX} Vlan1000
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel01
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel01
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel02
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel02
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel03
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel03
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel04
wait_until_iface_ready ${LAG_TABLE_PREFIX} PortChannel04