matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Finalize fast-reboot in warmboot finalizer (#14238)

Browse Source 
- Why I did it
To solve an issue with upgrade with fast-reboot including FW upgrade which has been introduced since moving to fast-reboot over warm-reboot infrastructure.
As well, this introduces fast-reboot finalizing logic to determine fast-reboot is done.

- How I did it
Added logic to finalize-warmboot script to handle fast-reboot as well, this makes sense as using fast-reboot over warm-reboot this script will be invoked. The script will clear fast-reboot entry from state-db instead of previous implementation that relied on timer. The timer could expire in some scenarios between fast-reboot finished causing fallback to cold-reboot and possible crashes.

As well this PR updates all services/scripts reading fast-reboot state-db entry to look for the updated value representing fast-reboot is active.

- How to verify it
Run fast-reboot and check that fast-reboot entry exists in state-db right after startup and being cleared as warm-reboot is finalized and not due to a timer.
This commit is contained in:
Aryeh Feigin 2023-04-09 16:59:15 +03:00 committed by GitHub
parent e32624d362
commit 41a9813018
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 64 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
files/image_config/ntp/ntp-config.sh
View File

@ -8,11 +8,11 @@ reboot_type='cold'
function get_database_reboot_type()
{
SYSTEM_WARM_START=`sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
SYSTEM_FAST_START=`sonic-db-cli STATE_DB get "FAST_REBOOT|system"`
SYSTEM_FAST_START=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_WARM_START}" == x"true" ]]; then
reboot_type='warm'
elif [[ x"${SYSTEM_FAST_START}" == x"1" ]]; then
elif [[ x"${SYSTEM_FAST_START}" == x"true" ]]; then
reboot_type='fast'
fi
}

41
files/image_config/warmboot-finalizer/finalize-warmboot.sh
View File

@ -52,6 +52,17 @@ function check_warm_boot()
WARM_BOOT=`sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
}
function check_fast_reboot()
{
debug "Checking if fast-reboot is enabled..."
FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${FAST_REBOOT}" == x"true" ]]; then
debug "Fast-reboot is enabled..."
else
debug "Fast-reboot is disabled..."
fi
}
function wait_for_database_service()
{
@ -97,6 +108,12 @@ function finalize_warm_boot()
sudo config warm_restart disable
}
function finalize_fast_reboot()
{
debug "Finalizing fast-reboot..."
sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null
}
function stop_control_plane_assistant()
{
if [[ -x ${ASSISTANT_SCRIPT} ]]; then
@ -119,14 +136,20 @@ function restore_counters_folder()
wait_for_database_service
check_fast_reboot
check_warm_boot
if [[ x"${WARM_BOOT}" != x"true" ]]; then
debug "warmboot is not enabled ..."
exit 0
if [[ x"${FAST_REBOOT}" != x"true" ]]; then
debug "fastboot is not enabled ..."
exit 0
fi
fi
restore_counters_folder
if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
restore_counters_folder
fi
get_component_list
@ -143,14 +166,22 @@ for i in `seq 60`; do
sleep 5
done
stop_control_plane_assistant
if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
stop_control_plane_assistant
fi
# Save DB after stopped control plane assistant to avoid extra entries
debug "Save in-memory database after warm reboot ..."
debug "Save in-memory database after warm/fast reboot ..."
config save -y
if [[ -n "${list}" ]]; then
debug "Some components didn't finish reconcile: ${list} ..."
fi
finalize_warm_boot
if [ x"${FAST_REBOOT}" == x"true" ]; then
finalize_fast_reboot
fi
if [ x"${WARM_BOOT}" == x"true" ]; then
finalize_warm_boot
fi

3
files/scripts/bgp.sh
View File

@ -30,7 +30,8 @@ function validate_restore_count()
function check_fast_boot ()
{
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
FAST_BOOT="true"
else
FAST_BOOT="false"

3
files/scripts/service_mgmt.sh
View File

@ -19,7 +19,8 @@ function check_warm_boot()
function check_fast_boot ()
{
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
FAST_BOOT="true"
else
FAST_BOOT="false"

7
files/scripts/swss.sh
View File

@ -60,7 +60,8 @@ function check_warm_boot()
function check_fast_boot()
{
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
FAST_BOOT="true"
else
FAST_BOOT="false"
@ -284,8 +285,8 @@ stop() {
# encountered error, e.g. syncd crashed. And swss needs to
# be restarted.
if [[ x"$FAST_BOOT" != x"true" ]]; then
debug "Clearing FAST_REBOOT flag..."
clean_up_tables STATE_DB "'FAST_REBOOT*'"
debug "Clearing FAST_RESTART_ENABLE_TABLE flag..."
sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false"
fi
# Unlock has to happen before reaching out to peer service
unlock_service_state_change

6
files/scripts/syncd_common.sh
View File

@ -50,7 +50,8 @@ function check_warm_boot()
function check_fast_boot()
{
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
FAST_BOOT="true"
else
FAST_BOOT="false"
@ -82,7 +83,8 @@ function getBootType()
;;
*SONIC_BOOT_TYPE=fast*|*fast-reboot*)
# check that the key exists
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
TYPE='fast'
else
TYPE='cold'

3
files/scripts/teamd.sh
View File

@ -32,7 +32,8 @@ function validate_restore_count()
function check_fast_boot ()
{
if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
FAST_BOOT="true"
else
FAST_BOOT="false"

22
src/sonic-py-common/sonic_py_common/device_info.py
View File

@ -699,14 +699,16 @@ def is_warm_restart_enabled(container_name):
# Check if System fast reboot is enabled.
def is_fast_reboot_enabled():
fb_system_state = 0
cmd = ['sonic-db-cli', 'STATE_DB', 'get', "FAST_REBOOT|system"]
proc = subprocess.Popen(cmd, universal_newlines=True, stdout=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
state_db = SonicV2Connector(host='127.0.0.1')
state_db.connect(state_db.STATE_DB, False)
TABLE_NAME_SEPARATOR = '|'
prefix = 'FAST_RESTART_ENABLE_TABLE' + TABLE_NAME_SEPARATOR
# Get the system warm reboot enable state
_hash = '{}{}'.format(prefix, 'system')
fb_system_state = state_db.get(state_db.STATE_DB, _hash, "enable")
fb_enable_state = True if fb_system_state == "true" else False
if proc.returncode != 0:
log.log_error("Error running command '{}'".format(cmd))
elif stdout:
fb_system_state = stdout.rstrip('\n')
return fb_system_state
state_db.close(state_db.STATE_DB)
return fb_enable_state