[services] kill container on stop in warm/fast mode (#10511)

To optimize stop on warm boot, added kill for containers Use service "kill" in the shutdown path for fast and warm reboot. For all other reload methods, service "stop" is used. This is done to save time in shutdown path, and to overall improve the time spent in warm and fast reload. How - Use service_mgmt.sh to trigger common logic to initiate kill (fast/warm) or stop (cold) for database.sh, radv.sh, snmp.sh, telemetry.sh, mgmt-framework.sh Signed-off-by: Stepan Blyschak <stepanb@nvidia.com>, Vaibhav H D <vaibhav.dixit@microsoft.com>
2022-04-19 00:27:48 +03:00 · 2022-04-19 00:27:48 +03:00 · fa1e364f54
commit fa1e364f54
parent 85447401c7
12 changed files with 115 additions and 100 deletions
--- a/files/build_templates/docker_image_ctl.j2
+++ b/files/build_templates/docker_image_ctl.j2
@ -31,12 +31,11 @@ function updateSyslogConf()
 {
    # On multiNPU platforms, change the syslog target ip to docker0 ip to allow logs from containers
    # running on the namespace to reach the rsyslog service running on the host
-    # Also update the container name  
+    # Also update the container name
    if [[ ($NUM_ASIC -gt 1) ]]; then
        TARGET_IP=$(docker network inspect bridge --format={{ "'{{(index .IPAM.Config 0).Gateway}}'" }})
        CONTAINER_NAME="$DOCKERNAME"
        TMP_FILE="/tmp/rsyslog.$CONTAINER_NAME.conf"
-
        sonic-cfggen -t /usr/share/sonic/templates/rsyslog-container.conf.j2 -a "{\"target_ip\": \"$TARGET_IP\", \"container_name\": \"$CONTAINER_NAME\" }"  > $TMP_FILE
        docker cp $TMP_FILE ${DOCKERNAME}:/etc/rsyslog.conf
        rm -rf $TMP_FILE
@ -437,6 +436,17 @@ stop() {
    {%- endif %}
 }

+kill() {
+    {%- if docker_container_name == "database" %}
+    docker kill $DOCKERNAME
+    if [ "$DEV" ]; then
+        ip netns delete "$NET_NS"
+    fi
+    {%- else %}
+    /usr/local/bin/container kill $DOCKERNAME
+    {%- endif %}
+}
+
 DOCKERNAME={{docker_container_name}}
 OP=$1
 DEV=$2 # namespace/device number to operate on
@ -464,7 +474,7 @@ fi
 [ -f /etc/sonic/sonic-environment ] && . /etc/sonic/sonic-environment

 case "$1" in
-    start|wait|stop)
+    start|wait|stop|kill)
        $1
        ;;
    *)
--- a/files/build_templates/mgmt-framework.service.j2
+++ b/files/build_templates/mgmt-framework.service.j2
@ -8,6 +8,6 @@ Before=ntp-config.service

 [Service]
 User={{ sonicadmin_user }}
-ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
-ExecStart=/usr/bin/{{docker_container_name}}.sh wait
-ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+ExecStartPre=/usr/local/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/local/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/local/bin/{{docker_container_name}}.sh stop
--- a/files/build_templates/per_namespace/database.service.j2
+++ b/files/build_templates/per_namespace/database.service.j2
@ -14,9 +14,9 @@ StartLimitBurst=3

 [Service]
 User=root
-ExecStartPre=/usr/bin/{{docker_container_name}}.sh start{% if multi_instance == 'true' %} %i{% endif %}
-ExecStart=/usr/bin/{{docker_container_name}}.sh wait{% if multi_instance == 'true' %} %i{% endif %}
-ExecStop=/usr/bin/{{docker_container_name}}.sh stop{% if multi_instance == 'true' %} %i{% endif %}
+ExecStartPre=/usr/local/bin/{{docker_container_name}}.sh start{% if multi_instance == 'true' %} %i{% endif %}
+ExecStart=/usr/local/bin/{{docker_container_name}}.sh wait{% if multi_instance == 'true' %} %i{% endif %}
+ExecStop=/usr/local/bin/{{docker_container_name}}.sh stop{% if multi_instance == 'true' %} %i{% endif %}
 Restart=always
 RestartSec=30

--- a/files/build_templates/snmp.service.j2
+++ b/files/build_templates/snmp.service.j2
@ -10,8 +10,8 @@ StartLimitIntervalSec=1200
 StartLimitBurst=3

 [Service]
-ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
-ExecStart=/usr/bin/{{docker_container_name}}.sh wait
-ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+ExecStartPre=/usr/local/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/local/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/local/bin/{{docker_container_name}}.sh stop
 Restart=always
 RestartSec=30
--- a/files/build_templates/sonic_debian_extension.j2
+++ b/files/build_templates/sonic_debian_extension.j2
@ -630,7 +630,7 @@ fi
 ## ensure proc is mounted
 sudo mount proc /proc -t proc || true
 if [[ $CONFIGURED_ARCH == armhf ]]; then
-    # A workaround to fix the armhf build hung issue, caused by sonic-platform-nokia-7215_1.0_armhf.deb post installation script 
+    # A workaround to fix the armhf build hung issue, caused by sonic-platform-nokia-7215_1.0_armhf.deb post installation script
    ps -eo pid,cmd | grep python | grep "/etc/entropy.py" | awk '{print $1}' | xargs sudo kill -9 2>/dev/null || true
 fi

@ -713,6 +713,10 @@ sudo LANG=C cp $SCRIPTS_DIR/gbsyncd.sh $FILESYSTEM_ROOT/usr/local/bin/gbsyncd.sh
 sudo LANG=C cp $SCRIPTS_DIR/bgp.sh $FILESYSTEM_ROOT/usr/local/bin/bgp.sh
 sudo LANG=C cp $SCRIPTS_DIR/teamd.sh $FILESYSTEM_ROOT/usr/local/bin/teamd.sh
 sudo LANG=C cp $SCRIPTS_DIR/radv.sh $FILESYSTEM_ROOT/usr/local/bin/radv.sh
+sudo LANG=C cp $SCRIPTS_DIR/database.sh $FILESYSTEM_ROOT/usr/local/bin/database.sh
+sudo LANG=C cp $SCRIPTS_DIR/snmp.sh $FILESYSTEM_ROOT/usr/local/bin/snmp.sh
+sudo LANG=C cp $SCRIPTS_DIR/telemetry.sh $FILESYSTEM_ROOT/usr/local/bin/telemetry.sh
+sudo LANG=C cp $SCRIPTS_DIR/mgmt-framework.sh $FILESYSTEM_ROOT/usr/local/bin/mgmt-framework.sh

 # Copy sonic-netns-exec script
 sudo LANG=C cp $SCRIPTS_DIR/sonic-netns-exec $FILESYSTEM_ROOT/usr/bin/sonic-netns-exec
--- a/files/build_templates/telemetry.service.j2
+++ b/files/build_templates/telemetry.service.j2
@ -10,8 +10,8 @@ StartLimitBurst=3

 [Service]
 User={{ sonicadmin_user }}
-ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
-ExecStart=/usr/bin/{{docker_container_name}}.sh wait
-ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+ExecStartPre=/usr/local/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/local/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/local/bin/{{docker_container_name}}.sh stop
 Restart=always
 RestartSec=30
--- a/files/scripts/database.sh
+++ b/files/scripts/database.sh
@ -0,0 +1 @@
+service_mgmt.sh
--- a/files/scripts/mgmt-framework.sh
+++ b/files/scripts/mgmt-framework.sh
@ -0,0 +1 @@
+service_mgmt.sh
--- a/files/scripts/radv.sh
+++ b/files/scripts/radv.sh
@ -1,84 +0,0 @@
-#!/bin/bash
-
-function debug()
-{
-    /usr/bin/logger $1
-    /bin/echo `date` "- $1" >> ${DEBUGLOG}
-}
-
-function check_warm_boot()
-{
-    SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
-    SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
-    if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then
-        WARM_BOOT="true"
-    else
-        WARM_BOOT="false"
-    fi
-}
-
-function check_fast_boot ()
-{
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
-        FAST_BOOT="true"
-    else
-        FAST_BOOT="false"
-    fi
-}
-
-start() {
-    debug "Starting ${SERVICE}$DEV service..."
-
-    check_warm_boot
-    check_fast_boot
-    debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}."
-    debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}."
-
-    # start service docker
-    /usr/bin/${SERVICE}.sh start $DEV
-    debug "Started ${SERVICE}$DEV service..."
-}
-
-wait() {
-    /usr/bin/${SERVICE}.sh wait $DEV
-}
-
-stop() {
-    debug "Stopping ${SERVICE}$DEV service..."
-
-    check_warm_boot
-    check_fast_boot
-    debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}."
-    debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}."
-
-    # For WARM/FAST boot do not perform service stop
-    if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then
-        /usr/bin/${SERVICE}.sh stop $DEV
-        debug "Stopped ${SERVICE}$DEV service..."
-    else
-        debug "Killing Docker radv..."
-        /usr/bin/docker kill radv &> /dev/null || debug "Docker radv is not running ($?) ..."
-    fi
-}
-
-DEV=$2
-
-SERVICE="radv"
-DEBUGLOG="/tmp/radv-debug$DEV.log"
-NAMESPACE_PREFIX="asic"
-if [ "$DEV" ]; then
-    NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
-    SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
-else
-    SONIC_DB_CLI="sonic-db-cli"
-fi
-
-case "$1" in
-    start|wait|stop)
-        $1
-        ;;
-    *)
-        echo "Usage: $0 {start|wait|stop}"
-        exit 1
-        ;;
-esac
--- a/files/scripts/radv.sh
+++ b/files/scripts/radv.sh
@ -0,0 +1 @@
+service_mgmt.sh
--- a/files/scripts/service_mgmt.sh
+++ b/files/scripts/service_mgmt.sh
@ -0,0 +1,80 @@
+#!/bin/bash
+
+function debug()
+{
+    /usr/bin/logger $1
+    /bin/echo `date` "- $1" >> ${DEBUGLOG}
+}
+
+function check_warm_boot()
+{
+    SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
+    SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
+    if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then
+        WARM_BOOT="true"
+    else
+        WARM_BOOT="false"
+    fi
+}
+
+function check_fast_boot ()
+{
+    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+        FAST_BOOT="true"
+    else
+        FAST_BOOT="false"
+    fi
+}
+
+start() {
+    debug "Starting ${SERVICE}$DEV service..."
+
+    # start service docker
+    /usr/bin/${SERVICE}.sh start $DEV
+    debug "Started ${SERVICE}$DEV service..."
+}
+
+wait() {
+    /usr/bin/${SERVICE}.sh wait $DEV
+}
+
+stop() {
+    debug "Stopping ${SERVICE}$DEV service..."
+
+    check_warm_boot
+    check_fast_boot
+    debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}."
+    debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}."
+
+    # For WARM/FAST boot do not perform service stop
+    if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then
+        /usr/bin/${SERVICE}.sh stop $DEV
+        debug "Stopped ${SERVICE}$DEV service..."
+    else
+        debug "Killing Docker ${SERVICE}${DEV}..."
+        /usr/bin/${SERVICE}.sh kill $DEV
+    fi
+}
+
+DEV=$2
+
+SCRIPT_NAME=$(basename -- "$0")
+SERVICE="${SCRIPT_NAME%.*}"
+DEBUGLOG="/tmp/$SERVICE-debug$DEV.log"
+NAMESPACE_PREFIX="asic"
+if [ "$DEV" ]; then
+    NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
+    SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
+else
+    SONIC_DB_CLI="sonic-db-cli"
+fi
+
+case "$1" in
+    start|wait|stop)
+        $1
+        ;;
+    *)
+        echo "Usage: $0 {start|wait|stop}"
+        exit 1
+        ;;
+esac
--- a/files/scripts/snmp.sh
+++ b/files/scripts/snmp.sh
@ -0,0 +1 @@
+service_mgmt.sh
--- a/files/scripts/telemetry.sh
+++ b/files/scripts/telemetry.sh
@ -0,0 +1 @@
+service_mgmt.sh