From 0b836390681488c8831f9ff1c62fab0e49b58541 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Thu, 24 Aug 2023 01:58:24 -0700 Subject: [PATCH] Fix CONFIG_DB_INITIALIZED flag check logic and set/reset flag for warmboot (#15685) (#16217) Cherypick of #15685 MSFT ADO: 24274591 Why I did it Two changes: 1 Fix a day1 issue, where check to wait until CONFIG_DB_INITIALIZED is incorrect. There are multiple places where same incorrect logic is used. Current logic (until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];) will always result in pass, irrespective of the result of GET operation. root@str2-7060cx-32s-29:~# sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED" 1 root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; do echo "entered here"; done root@str2-7060cx-32s-29:~# root@str2-7060cx-32s-29:~# root@str2-7060cx-32s-29:~# sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED" 0 root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; do echo "entered here"; done root@str2-7060cx-32s-29:~# Fix this logic by checking for value of flag to be "1". root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do echo "entered here"; done entered here entered here entered here This gap in logic was highlighted when another fix was merged: #14933 The issue being fixed here caused warmboot-finalizer to not wait until config-db is initialized. 2 Set and unset CONFIG_DB_INITIALIZED for warm-reboot case Currently, during warm shutdown CONFIG_DB_INITIALIZED's value is stored in redis db backup. This is restored back when the dump is loaded during warm-recovery. So the value of CONFIG_DB_INITIALIZED does not depend on config db's state, however it remain what it was before reboot. Fix this by setting CONFIG_DB_INITIALIZED to 0 as when the DB is loaded, and set it to 1 after db_migrator is done. Work item tracking Microsoft ADO (number only): How I did it How to verify it --- files/build_templates/docker_image_ctl.j2 | 27 +++++++------------ files/image_config/config-setup/config-setup | 4 ++- .../warmboot-finalizer/finalize-warmboot.sh | 2 +- files/scripts/swss.sh | 2 +- files/scripts/syncd_common.sh | 2 +- 5 files changed, 15 insertions(+), 22 deletions(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 01a52fb7a0..23d111051f 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -235,6 +235,7 @@ function postStartAction() ($(docker exec -i database$DEV sonic-db-cli PING | grep -c PONG) -gt 0) ]]; do sleep 1; done + if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast" || "$BOOT_TYPE" == "fast") && -f $WARM_DIR/dump.rdb ]]; then # retain the dump file from last boot for debugging purposes mv $WARM_DIR/dump.rdb $WARM_DIR/dump.rdb.old @@ -248,28 +249,18 @@ function postStartAction() $SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db fi fi - - if [[ "$BOOT_TYPE" == "fast" ]]; then - # set the key to expire in 3 minutes - $SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" - fi - - $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi - if [ -e /tmp/pending_config_migration ]; then + if [ -e /tmp/pending_config_migration ] || [ -e /tmp/pending_config_initialization ]; then # this is first boot to a new image, config-setup execution is pending. - # For fast/cold reboot case, DB contains nothing at this point - # Call db_migrator after config-setup loads the config (from old config or minigraph) - echo "Delaying db_migrator until config migration is over" + # for warmboot case, DB is loaded but migration is still pending + # For firstbboot/fast/cold reboot case, DB contains nothing at this point + # unset CONFIG_DB_INITIALIZED to indicate pending config load and migration + # This flag will be set to "1" after DB migration/initialization is completed as part of config-setup + $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0" else - # this is not a first time boot to a new image. Datbase container starts w/ old pre-existing config - if [[ -x /usr/local/bin/db_migrator.py ]]; then - # Migrate the DB to the latest schema version if needed - if [ -z "$DEV" ]; then - /usr/local/bin/db_migrator.py -o migrate - fi - fi + # set CONFIG_DB_INITIALIZED to indicate end of config load and migration + $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi # Add redis UDS to the redis group and give read/write access to the group REDIS_SOCK="/var/run/redis${DEV}/redis.sock" diff --git a/files/image_config/config-setup/config-setup b/files/image_config/config-setup/config-setup index 28a3a4373a..d4826d9b62 100755 --- a/files/image_config/config-setup/config-setup +++ b/files/image_config/config-setup/config-setup @@ -256,6 +256,7 @@ do_config_initialization() fi rm -f /tmp/pending_config_initialization + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" } # Restore config-setup post migration hooks from a backup copy @@ -305,13 +306,14 @@ check_all_config_db_present() } # DB schema is subject to change between two images -# Perform DB schema migration after loading backup config from previous image +# Perform DB schema migration after loading backup config/minigraph from previous image do_db_migration() { if [[ -x /usr/local/bin/db_migrator.py ]]; then # Migrate the DB to the latest schema version if needed /usr/local/bin/db_migrator.py -o migrate fi + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" } # Perform configuration migration from backup copy. diff --git a/files/image_config/warmboot-finalizer/finalize-warmboot.sh b/files/image_config/warmboot-finalizer/finalize-warmboot.sh index 783f388181..b01832a6c7 100755 --- a/files/image_config/warmboot-finalizer/finalize-warmboot.sh +++ b/files/image_config/warmboot-finalizer/finalize-warmboot.sh @@ -74,7 +74,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done diff --git a/files/scripts/swss.sh b/files/scripts/swss.sh index 2974104e66..6b3e60b61f 100755 --- a/files/scripts/swss.sh +++ b/files/scripts/swss.sh @@ -87,7 +87,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done } diff --git a/files/scripts/syncd_common.sh b/files/scripts/syncd_common.sh index a850e31b20..826cdd731b 100755 --- a/files/scripts/syncd_common.sh +++ b/files/scripts/syncd_common.sh @@ -66,7 +66,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done }