2018-09-24 18:35:01 -05:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
|
|
SERVICE="syncd"
|
|
|
|
PEER="swss"
|
|
|
|
DEBUGLOG="/tmp/swss-syncd-debug.log"
|
|
|
|
LOCKFILE="/tmp/swss-syncd-lock"
|
|
|
|
|
|
|
|
function debug()
|
|
|
|
{
|
2019-02-10 13:56:31 -06:00
|
|
|
/usr/bin/logger $1
|
2018-09-24 18:35:01 -05:00
|
|
|
/bin/echo `date` "- $1" >> ${DEBUGLOG}
|
|
|
|
}
|
|
|
|
|
|
|
|
function lock_service_state_change()
|
|
|
|
{
|
|
|
|
debug "Locking ${LOCKFILE} from ${SERVICE} service"
|
|
|
|
|
|
|
|
exec {LOCKFD}>${LOCKFILE}
|
|
|
|
/usr/bin/flock -x ${LOCKFD}
|
|
|
|
trap "/usr/bin/flock -u ${LOCKFD}" 0 2 3 15
|
|
|
|
|
|
|
|
debug "Locked ${LOCKFILE} (${LOCKFD}) from ${SERVICE} service"
|
|
|
|
}
|
|
|
|
|
|
|
|
function unlock_service_state_change()
|
|
|
|
{
|
|
|
|
debug "Unlocking ${LOCKFILE} (${LOCKFD}) from ${SERVICE} service"
|
|
|
|
/usr/bin/flock -u ${LOCKFD}
|
|
|
|
}
|
|
|
|
|
|
|
|
function check_warm_boot()
|
|
|
|
{
|
2019-02-14 14:12:55 -06:00
|
|
|
SYSTEM_WARM_START=`/usr/bin/redis-cli -n 6 hget "WARM_RESTART_ENABLE_TABLE|system" enable`
|
|
|
|
SERVICE_WARM_START=`/usr/bin/redis-cli -n 6 hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
|
2018-09-24 18:35:01 -05:00
|
|
|
# SYSTEM_WARM_START could be empty, always make WARM_BOOT meaningful.
|
2018-10-16 13:20:39 -05:00
|
|
|
if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then
|
2018-09-24 18:35:01 -05:00
|
|
|
WARM_BOOT="true"
|
|
|
|
else
|
|
|
|
WARM_BOOT="false"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
function wait_for_database_service()
|
|
|
|
{
|
|
|
|
# Wait for redis server start before database clean
|
|
|
|
until [[ $(/usr/bin/docker exec database redis-cli ping | grep -c PONG) -gt 0 ]];
|
|
|
|
do sleep 1;
|
|
|
|
done
|
|
|
|
|
|
|
|
# Wait for configDB initialization
|
|
|
|
until [[ $(/usr/bin/docker exec database redis-cli -n 4 GET "CONFIG_DB_INITIALIZED") ]];
|
|
|
|
do sleep 1;
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
2018-12-15 13:36:12 -06:00
|
|
|
function getBootType()
|
|
|
|
{
|
2019-06-27 00:02:21 -05:00
|
|
|
# same code snippet in files/build_templates/docker_image_ctl.j2
|
|
|
|
case "$(cat /proc/cmdline)" in
|
|
|
|
*SONIC_BOOT_TYPE=warm*)
|
2018-12-15 13:36:12 -06:00
|
|
|
TYPE='warm'
|
|
|
|
;;
|
2019-06-27 00:02:21 -05:00
|
|
|
*SONIC_BOOT_TYPE=fastfast*)
|
2018-12-15 13:36:12 -06:00
|
|
|
TYPE='fastfast'
|
|
|
|
;;
|
2019-06-27 00:02:21 -05:00
|
|
|
*SONIC_BOOT_TYPE=fast*|*fast-reboot*)
|
2019-12-13 08:07:13 -06:00
|
|
|
# check that the key exists
|
|
|
|
if [[ $(redis-cli -n 6 GET "FAST_REBOOT|system") == "1" ]]; then
|
|
|
|
TYPE='fast'
|
|
|
|
else
|
|
|
|
TYPE='cold'
|
|
|
|
fi
|
2018-12-15 13:36:12 -06:00
|
|
|
;;
|
|
|
|
*)
|
|
|
|
TYPE='cold'
|
|
|
|
esac
|
|
|
|
echo "${TYPE}"
|
|
|
|
}
|
|
|
|
|
2018-09-24 18:35:01 -05:00
|
|
|
start() {
|
|
|
|
debug "Starting ${SERVICE} service..."
|
|
|
|
|
|
|
|
lock_service_state_change
|
|
|
|
|
2018-10-01 21:01:04 -05:00
|
|
|
mkdir -p /host/warmboot
|
|
|
|
|
2018-09-24 18:35:01 -05:00
|
|
|
wait_for_database_service
|
|
|
|
check_warm_boot
|
|
|
|
|
|
|
|
debug "Warm boot flag: ${SERVICE} ${WARM_BOOT}."
|
|
|
|
|
2018-11-21 19:56:19 -06:00
|
|
|
if [[ x"$WARM_BOOT" == x"true" ]]; then
|
|
|
|
# Leave a mark for syncd scripts running inside docker.
|
|
|
|
touch /host/warmboot/warm-starting
|
|
|
|
else
|
|
|
|
rm -f /host/warmboot/warm-starting
|
2018-09-24 18:35:01 -05:00
|
|
|
fi
|
|
|
|
|
2018-12-15 13:36:12 -06:00
|
|
|
# platform specific tasks
|
2019-01-10 16:09:04 -06:00
|
|
|
|
|
|
|
# start mellanox drivers regardless of
|
|
|
|
# boot type
|
2018-12-15 13:36:12 -06:00
|
|
|
if [ x"$sonic_asic_platform" == x"mellanox" ]; then
|
|
|
|
BOOT_TYPE=`getBootType`
|
|
|
|
if [[ x"$WARM_BOOT" == x"true" || x"$BOOT_TYPE" == x"fast" ]]; then
|
|
|
|
export FAST_BOOT=1
|
|
|
|
fi
|
|
|
|
/usr/bin/mst start
|
|
|
|
/usr/bin/mlnx-fw-upgrade.sh
|
|
|
|
/etc/init.d/sxdkernel start
|
|
|
|
/sbin/modprobe i2c-dev
|
|
|
|
fi
|
|
|
|
|
2019-01-10 16:09:04 -06:00
|
|
|
if [[ x"$WARM_BOOT" != x"true" ]]; then
|
|
|
|
if [ x$sonic_asic_platform == x'cavium' ]; then
|
|
|
|
/etc/init.d/xpnet.sh start
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
2018-09-24 18:35:01 -05:00
|
|
|
# start service docker
|
|
|
|
/usr/bin/${SERVICE}.sh start
|
|
|
|
debug "Started ${SERVICE} service..."
|
|
|
|
|
|
|
|
unlock_service_state_change
|
2019-03-02 17:28:34 -06:00
|
|
|
}
|
|
|
|
|
2019-03-08 12:59:41 -06:00
|
|
|
wait() {
|
|
|
|
/usr/bin/${SERVICE}.sh wait
|
2018-09-24 18:35:01 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
stop() {
|
|
|
|
debug "Stopping ${SERVICE} service..."
|
|
|
|
|
|
|
|
lock_service_state_change
|
|
|
|
check_warm_boot
|
|
|
|
debug "Warm boot flag: ${SERVICE} ${WARM_BOOT}."
|
|
|
|
|
2018-10-01 21:01:04 -05:00
|
|
|
if [[ x"$WARM_BOOT" == x"true" ]]; then
|
2018-11-15 17:47:33 -06:00
|
|
|
TYPE=warm
|
|
|
|
else
|
|
|
|
TYPE=cold
|
|
|
|
fi
|
2018-10-01 21:01:04 -05:00
|
|
|
|
2018-12-15 13:36:12 -06:00
|
|
|
if [[ x$sonic_asic_platform != x"mellanox" ]] || [[ x$TYPE != x"cold" ]]; then
|
|
|
|
debug "${TYPE} shutdown syncd process ..."
|
|
|
|
/usr/bin/docker exec -i syncd /usr/bin/syncd_request_shutdown --${TYPE}
|
2018-10-01 21:01:04 -05:00
|
|
|
|
[docker-syncd] Add timeout to force stop syncd container (#4617)
**- Why I did it**
When I tested auto-restart feature of swss container by manually killing one of critical processes in it, swss will be stopped. Then syncd container as the peer container should also be
stopped as expected. However, I found sometimes syncd container can be stopped, sometimes
it can not be stopped. The reason why syncd container can not be stopped is the process
(/usr/local/bin/syncd.sh stop) to execute the stop() function will be stuck between the lines 164 –167. Systemd will wait for 90 seconds and then kill this process.
164 # wait until syncd quit gracefully
165 while docker top syncd$DEV | grep -q /usr/bin/syncd; do
166 sleep 0.1
167 done
The first thing I did is to profile how long this while loop will spin if syncd container can be
normally stopped after swss container is stopped. The result is 5 seconds or 6 seconds. If syncd
container can be normally stopped, two messages will be written into syslog:
str-a7050-acs-3 NOTICE syncd#dsserve: child /usr/bin/syncd exited status: 134
str-a7050-acs-3 INFO syncd#supervisord: syncd [5] child /usr/bin/syncd exited status: 134
The second thing I did was to add a timer in the condition of while loop to ensure this while loop will be forced to exit after 20 seconds:
After that, the testing result is that syncd container can be normally stopped if swss is stopped
first. One more thing I want to mention is that if syncd container is stopped during 5 seconds or 6 seconds, then the two log messages can be still seen in syslog. However, if the execution
time of while loop is longer than 20 seconds and is forced to exit, although syncd container can be stopped, I did not see these two messages in syslog. Further, although I observed the auto-restart feature of swss container can work correctly right now, I can not make sure the issue which syncd container can not stopped will occur in future.
**- How I did it**
I added a timer around the while loop in stop() function. This while loop will exit after spinning
20 seconds.
Signed-off-by: Yong Zhao <yozhao@microsoft.com>
2020-06-04 17:17:28 -05:00
|
|
|
# wait until syncd quits gracefully or force syncd to exit after
|
|
|
|
# waiting for 20 seconds
|
|
|
|
start_in_secs=${SECONDS}
|
|
|
|
end_in_secs=${SECONDS}
|
|
|
|
timer_threshold=20
|
|
|
|
while docker top syncd | grep -q /usr/bin/syncd \
|
|
|
|
&& [[ $((end_in_secs - start_in_secs)) -le $timer_threshold ]]; do
|
2018-12-15 13:36:12 -06:00
|
|
|
sleep 0.1
|
[docker-syncd] Add timeout to force stop syncd container (#4617)
**- Why I did it**
When I tested auto-restart feature of swss container by manually killing one of critical processes in it, swss will be stopped. Then syncd container as the peer container should also be
stopped as expected. However, I found sometimes syncd container can be stopped, sometimes
it can not be stopped. The reason why syncd container can not be stopped is the process
(/usr/local/bin/syncd.sh stop) to execute the stop() function will be stuck between the lines 164 –167. Systemd will wait for 90 seconds and then kill this process.
164 # wait until syncd quit gracefully
165 while docker top syncd$DEV | grep -q /usr/bin/syncd; do
166 sleep 0.1
167 done
The first thing I did is to profile how long this while loop will spin if syncd container can be
normally stopped after swss container is stopped. The result is 5 seconds or 6 seconds. If syncd
container can be normally stopped, two messages will be written into syslog:
str-a7050-acs-3 NOTICE syncd#dsserve: child /usr/bin/syncd exited status: 134
str-a7050-acs-3 INFO syncd#supervisord: syncd [5] child /usr/bin/syncd exited status: 134
The second thing I did was to add a timer in the condition of while loop to ensure this while loop will be forced to exit after 20 seconds:
After that, the testing result is that syncd container can be normally stopped if swss is stopped
first. One more thing I want to mention is that if syncd container is stopped during 5 seconds or 6 seconds, then the two log messages can be still seen in syslog. However, if the execution
time of while loop is longer than 20 seconds and is forced to exit, although syncd container can be stopped, I did not see these two messages in syslog. Further, although I observed the auto-restart feature of swss container can work correctly right now, I can not make sure the issue which syncd container can not stopped will occur in future.
**- How I did it**
I added a timer around the while loop in stop() function. This while loop will exit after spinning
20 seconds.
Signed-off-by: Yong Zhao <yozhao@microsoft.com>
2020-06-04 17:17:28 -05:00
|
|
|
end_in_secs=${SECONDS}
|
2018-12-15 13:36:12 -06:00
|
|
|
done
|
2018-11-15 17:47:33 -06:00
|
|
|
|
[docker-syncd] Add timeout to force stop syncd container (#4617)
**- Why I did it**
When I tested auto-restart feature of swss container by manually killing one of critical processes in it, swss will be stopped. Then syncd container as the peer container should also be
stopped as expected. However, I found sometimes syncd container can be stopped, sometimes
it can not be stopped. The reason why syncd container can not be stopped is the process
(/usr/local/bin/syncd.sh stop) to execute the stop() function will be stuck between the lines 164 –167. Systemd will wait for 90 seconds and then kill this process.
164 # wait until syncd quit gracefully
165 while docker top syncd$DEV | grep -q /usr/bin/syncd; do
166 sleep 0.1
167 done
The first thing I did is to profile how long this while loop will spin if syncd container can be
normally stopped after swss container is stopped. The result is 5 seconds or 6 seconds. If syncd
container can be normally stopped, two messages will be written into syslog:
str-a7050-acs-3 NOTICE syncd#dsserve: child /usr/bin/syncd exited status: 134
str-a7050-acs-3 INFO syncd#supervisord: syncd [5] child /usr/bin/syncd exited status: 134
The second thing I did was to add a timer in the condition of while loop to ensure this while loop will be forced to exit after 20 seconds:
After that, the testing result is that syncd container can be normally stopped if swss is stopped
first. One more thing I want to mention is that if syncd container is stopped during 5 seconds or 6 seconds, then the two log messages can be still seen in syslog. However, if the execution
time of while loop is longer than 20 seconds and is forced to exit, although syncd container can be stopped, I did not see these two messages in syslog. Further, although I observed the auto-restart feature of swss container can work correctly right now, I can not make sure the issue which syncd container can not stopped will occur in future.
**- How I did it**
I added a timer around the while loop in stop() function. This while loop will exit after spinning
20 seconds.
Signed-off-by: Yong Zhao <yozhao@microsoft.com>
2020-06-04 17:17:28 -05:00
|
|
|
if [[ $((end_in_secs - start_in_secs)) -gt $timer_threshold ]]; then
|
|
|
|
debug "syncd process in container syncd$DEV did not exit gracefully"
|
|
|
|
fi
|
|
|
|
|
2018-12-15 13:36:12 -06:00
|
|
|
/usr/bin/docker exec -i syncd /bin/sync
|
|
|
|
debug "Finished ${TYPE} shutdown syncd process ..."
|
|
|
|
fi
|
2018-10-01 21:01:04 -05:00
|
|
|
|
2018-09-24 18:35:01 -05:00
|
|
|
/usr/bin/${SERVICE}.sh stop
|
|
|
|
debug "Stopped ${SERVICE} service..."
|
|
|
|
|
2019-01-10 16:09:04 -06:00
|
|
|
# platform specific tasks
|
|
|
|
|
2018-09-24 18:35:01 -05:00
|
|
|
if [[ x"$WARM_BOOT" != x"true" ]]; then
|
2019-02-15 18:08:08 -06:00
|
|
|
if [ x$sonic_asic_platform == x'mellanox' ]; then
|
|
|
|
/etc/init.d/sxdkernel stop
|
|
|
|
/usr/bin/mst stop
|
|
|
|
elif [ x$sonic_asic_platform == x'cavium' ]; then
|
2018-09-24 18:35:01 -05:00
|
|
|
/etc/init.d/xpnet.sh stop
|
|
|
|
/etc/init.d/xpnet.sh start
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
unlock_service_state_change
|
|
|
|
}
|
|
|
|
|
|
|
|
case "$1" in
|
2019-03-08 12:59:41 -06:00
|
|
|
start|wait|stop)
|
2018-09-24 18:35:01 -05:00
|
|
|
$1
|
|
|
|
;;
|
|
|
|
*)
|
2019-03-08 12:59:41 -06:00
|
|
|
echo "Usage: $0 {start|wait|stop}"
|
2018-09-24 18:35:01 -05:00
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
esac
|