[Mellanox] Modified Platform API to support all firmware updates in single boot (#9608)

Why I did it
Requirements from Microsoft for fwutil update all state that all firmwares which support this upgrade flow must support upgrade within a single boot cycle. This conflicted with a number of Mellanox upgrade flows which have been revised to safely meet this requirement.

How I did it
Added --no-power-cycle flags to SSD and ONIE firmware scripts
Modified Platform API to call firmware upgrade flows with this new flag during fwutil update all
Added a script to our reboot plugin to handle installing firmwares in the correct order with prior to reboot
How to verify it
Populate platform_components.json with firmware for CPLD / BIOS / ONIE / SSD
Execute fwutil update all fw --boot cold
CPLD will burn / ONIE and BIOS images will stage / SSD will schedule for reboot
Reboot the switch
SSD will install / CPLD will refresh / switch will power cycle into ONIE
ONIE installer will upgrade ONIE and BIOS / switch will reboot back into SONiC
In SONiC run fwutil show status to check that all firmware upgrades were successful

device/mellanox/x86_64-mlnx_msn2700-r0/platform_reboot

@@ -3,6 +3,7 @@
 declare -r EXIT_SUCCESS="0"
 declare -r EXIT_ERROR="1"
 
+declare -r PENDING_COMPONENT_FW="/usr/bin/install-pending-fw.py"
 declare -r FW_UPGRADE_SCRIPT="/usr/bin/mlnx-fw-upgrade.sh"
 declare -r SYSFS_PWR_CYCLE="/sys/devices/platform/mlxplat/mlxreg-io/hwmon/hwmon*/pwr_cycle"
 
@@ -40,4 +41,6 @@ if [[ "${EXIT_CODE}" != "${EXIT_SUCCESS}" ]]; then
     fi
 fi
 
+${PENDING_COMPONENT_FW}
+
 SafePwrCycle

files/build_templates/sonic_debian_extension.j2

@@ -842,6 +842,7 @@ sudo cp $files_path/$ISSU_VERSION_FILE $FILESYSTEM_ROOT/etc/mlnx/issu-version
 sudo cp $files_path/$MLNX_FFB_SCRIPT $FILESYSTEM_ROOT/usr/bin/mlnx-ffb.sh
 sudo cp $files_path/$MLNX_ONIE_FW_UPDATE $FILESYSTEM_ROOT/usr/bin/$MLNX_ONIE_FW_UPDATE
 sudo cp $files_path/$MLNX_SSD_FW_UPDATE $FILESYSTEM_ROOT/usr/bin/$MLNX_SSD_FW_UPDATE
+sudo cp $files_path/$MLNX_INSTALL_PENDING_FW $FILESYSTEM_ROOT/usr/bin/$MLNX_INSTALL_PENDING_FW
 j2 platform/mellanox/mlnx-fw-upgrade.j2 | sudo tee $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
 sudo chmod 755 $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
 

platform/mellanox/install-pending-fw.dep

@@ -0,0 +1,10 @@
+# DPKG FRK
+
+DPATH := $($(MLNX_INSTALL_PENDING_FW)_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/install-pending-fw.mk $(PLATFORM_PATH)/install-pending-fw.dep
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES += $(addprefix $(DPATH),$(MLNX_INSTALL_PENDING_FW))
+
+$(MLNX_INSTALL_PENDING_FW)_CACHE_MODE := GIT_CONTENT_SHA
+$(MLNX_INSTALL_PENDING_FW)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(MLNX_INSTALL_PENDING_FW)_DEP_FILES := $(DEP_FILES)

platform/mellanox/install-pending-fw.mk

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2020-2021 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Firmware pending update checker and installer
+
+MLNX_INSTALL_PENDING_FW = install-pending-fw.py
+$(MLNX_INSTALL_PENDING_FW)_PATH = $(PLATFORM_PATH)/
+SONIC_COPY_FILES += $(MLNX_INSTALL_PENDING_FW)
+
+MLNX_FILES += $(MLNX_INSTALL_PENDING_FW)
+
+export MLNX_INSTALL_PENDING_FW

platform/mellanox/install-pending-fw.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from fwutil.lib import ComponentStatusProvider, PlatformComponentsParser
+from sonic_platform.component import ComponentCPLD, MPFAManager
+
+# Globals
+FW_STATUS_SCHEDULED = "scheduled"
+CPLD_FLAG = False
+
+# Init platform chassis helper classes
+csp = ComponentStatusProvider()
+pcp = PlatformComponentsParser(csp.is_modular_chassis())
+
+# Parse update status file
+update_status = csp.read_au_status_file_if_exists()
+
+if update_status is None:
+    exit(0)
+
+# Parse platform components file
+try:
+    pcp.parse_platform_components()
+except Exception as e:
+    print("Error parsing platform components. Firmware update failed: {}".format(str(e)))
+    print("System will reboot in 10 seconds please fix issue and run update command again.")
+    time.sleep(10)
+    exit(-1)
+
+# Iterate each component in the status file
+comp_install = []
+files = []
+
+for boot_type, components in update_status.items():
+    for comp in components:
+
+        # Skip if fw isn't scheduled for install at reboot
+        if comp["info"] != FW_STATUS_SCHEDULED: continue
+
+        # Get component object and target firmware file
+        key = comp["comp"]
+        comp_path = key.split("/")
+
+        if len(comp_path) == 3:
+            # Module component
+            _, parent_name, comp_name = comp_path
+            fw_file = pcp.module_component_map[parent_name][comp_name]["firmware"]
+            component = csp.module_component_map[parent_name][comp_name]
+        else: 
+            # Chassis component
+            parent_name, comp_name = comp_path
+            fw_file = pcp.chassis_component_map[parent_name][comp_name]["firmware"]
+            component = csp.chassis_component_map[parent_name][comp_name]
+
+        # Install firmware. If CPLD flag to be installed last due to force reboot during refresh
+        if type(component) == ComponentCPLD:
+            if CPLD_FLAG:
+                # Only need one refresh
+                continue
+            mpfa = MPFAManager(fw_file)
+            mpfa.extract()
+            if not mpfa.get_metadata().has_option('firmware', 'refresh'):
+                print("Failed to get CPLD refresh firmware. Skipping.")
+                continue
+            CPLD_FLAG = True
+            refresh_firmware = mpfa.get_metadata().get('firmware', 'refresh')
+            comp_install = comp_install + [component]
+            files = files + [os.path.join(mpfa.get_path(), refresh_firmware)]
+        else:
+            comp_install = [component] + comp_install
+            files = [fw_file] + files
+
+# Do install
+for i, c in enumerate(comp_install):
+    try:
+        if type(c) == ComponentCPLD:
+            c.install_firmware(files[i])
+        else:
+            c.install_firmware(files[i], allow_reboot=False)
+    except Exception as e:
+        print("Firmware install for {} FAILED with: {}".format(c.get_name(),e))
+

platform/mellanox/mlnx-onie-fw-update.sh

@@ -186,7 +186,12 @@ case "${cmd}" in
             rc=$?
             disable_onie_access
             if [[ ${rc} -eq 0 ]]; then
-                system_reboot
+                if [[ "${arg}" == "--no-reboot" ]]; then
+                    echo "INFO: ONIE firmware update successfully STAGED for install at NEXT reboot. Please reboot manually to complete installation."
+                    exit 0
+                else
+                    system_reboot
+                fi
             else
                 echo "ERROR: failed to enable ONIE firmware update mode"
                 exit ${rc}

platform/mellanox/mlnx-platform-api/sonic_platform/component.py

@@ -34,16 +34,16 @@ try:
     else:
         import ConfigParser as configparser
 
+    from shutil import copyfile
+
     from sonic_platform_base.component_base import ComponentBase,           \
                                                     FW_AUTO_INSTALLED,      \
+                                                    FW_AUTO_UPDATED,        \
+                                                    FW_AUTO_SCHEDULED,      \
                                                     FW_AUTO_ERR_BOOT_TYPE,  \
-                                                    FW_AUTO_ERR_IMAGE
+                                                    FW_AUTO_ERR_IMAGE,      \
+                                                    FW_AUTO_ERR_UNKNOWN
 
-    # Temp workaround to fix build issue, shall be refactor once sonic-platform-common submodule pointer is updated
-    try:
-        from sonic_platform_base.component_base import FW_AUTO_ERR_UNKNOWN
-    except ImportError as e:
-        from sonic_platform_base.component_base import FW_AUTO_ERR_UKNOWN as FW_AUTO_ERR_UNKNOWN
 except ImportError as e:
     raise ImportError(str(e) + "- required module not found")
 
@@ -124,6 +124,7 @@ class ONIEUpdater(object):
     ONIE_FW_UPDATE_CMD_ADD = '/usr/bin/mlnx-onie-fw-update.sh add {}'
     ONIE_FW_UPDATE_CMD_REMOVE = '/usr/bin/mlnx-onie-fw-update.sh remove {}'
     ONIE_FW_UPDATE_CMD_UPDATE = '/usr/bin/mlnx-onie-fw-update.sh update'
+    ONIE_FW_UPDATE_CMD_INSTALL = '/usr/bin/mlnx-onie-fw-update.sh update --no-reboot'
     ONIE_FW_UPDATE_CMD_SHOW_PENDING = '/usr/bin/mlnx-onie-fw-update.sh show-pending'
 
     ONIE_VERSION_PARSE_PATTERN = '([0-9]{4})\.([0-9]{2})-([0-9]+)\.([0-9]+)\.([0-9]+)-([0-9]+)'
@@ -135,6 +136,18 @@ class ONIEUpdater(object):
 
     ONIE_IMAGE_INFO_COMMAND = '/bin/bash {} -q -i'
 
+    BIOS_UPDATE_FILE_EXT = '.rom'
+
+    def __add_prefix(self, image_path):
+        if self.BIOS_UPDATE_FILE_EXT not in image_path:
+            rename_path = "/tmp/00-{}".format(os.path.basename(image_path))
+        else:
+            rename_path = "/tmp/99-{}".format(os.path.basename(image_path))
+
+        copyfile(image_path, rename_path)
+
+        return rename_path
+
     def __mount_onie_fs(self):
         fs_mountpoint = '/mnt/onie-fs'
         onie_path = '/lib/onie'
@@ -172,7 +185,9 @@ class ONIEUpdater(object):
             os.rmdir(fs_mountpoint)
 
     def __stage_update(self, image_path):
-        cmd = self.ONIE_FW_UPDATE_CMD_ADD.format(image_path)
+        rename_path = self.__add_prefix(image_path)
+
+        cmd = self.ONIE_FW_UPDATE_CMD_ADD.format(rename_path)
 
         try:
             subprocess.check_call(cmd.split(), universal_newlines=True)
@@ -180,15 +195,20 @@ class ONIEUpdater(object):
             raise RuntimeError("Failed to stage firmware update: {}".format(str(e)))
 
     def __unstage_update(self, image_path):
-        cmd = self.ONIE_FW_UPDATE_CMD_REMOVE.format(os.path.basename(image_path))
+        rename_path = self.__add_prefix(image_path)
+
+        cmd = self.ONIE_FW_UPDATE_CMD_REMOVE.format(os.path.basename(rename_path))
 
         try:
             subprocess.check_call(cmd.split(), universal_newlines=True)
         except subprocess.CalledProcessError as e:
             raise RuntimeError("Failed to unstage firmware update: {}".format(str(e)))
 
-    def __trigger_update(self):
-        cmd = self.ONIE_FW_UPDATE_CMD_UPDATE
+    def __trigger_update(self, allow_reboot):
+        if allow_reboot:
+            cmd = self.ONIE_FW_UPDATE_CMD_UPDATE
+        else:
+            cmd = self.ONIE_FW_UPDATE_CMD_INSTALL
 
         try:
             subprocess.check_call(cmd.split(), universal_newlines=True)
@@ -205,7 +225,8 @@ class ONIEUpdater(object):
         except subprocess.CalledProcessError as e:
             raise RuntimeError("Failed to get pending firmware updates: {}".format(str(e)))
 
-        basename = os.path.basename(image_path)
+        rename_path = self.__add_prefix(image_path)
+        basename = os.path.basename(rename_path)
 
         for line in output.splitlines():
             if line.startswith(basename):
@@ -304,29 +325,11 @@ class ONIEUpdater(object):
 
         return firmware_info
 
-    def update_firmware(self, image_path):
-        cmd = self.ONIE_FW_UPDATE_CMD_SHOW_PENDING
-
-        try:
-            output = subprocess.check_output(cmd.split(),
-                                             stderr=subprocess.STDOUT,
-                                             universal_newlines=True).rstrip('\n')
-        except subprocess.CalledProcessError as e:
-            raise RuntimeError("Failed to get pending firmware updates: {}".format(str(e)))
-
-        no_pending_updates = False
-
-        for line in output.splitlines():
-            if line.startswith(self.ONIE_NO_PENDING_UPDATES_ATTR):
-                no_pending_updates = True
-                break
-
-        if not no_pending_updates:
-            raise RuntimeError("Failed to complete firmware update: pending updates are present")
+    def update_firmware(self, image_path, allow_reboot=True):
 
         try:
             self.__stage_update(image_path)
-            self.__trigger_update()
+            self.__trigger_update(allow_reboot)
         except:
             if self.__is_update_staged(image_path):
                 self.__unstage_update(image_path)
@@ -364,22 +367,21 @@ class Component(ComponentBase):
         if boot_action is fast.
         """
 
-        default_supported_boot = ['cold']
-
         # Verify image path exists
         if not os.path.exists(image_path):
             # Invalid image path
             return FW_AUTO_ERR_IMAGE
 
-        if boot_action in default_supported_boot:
-            if self.update_firmware(image_path):
-                # Successful update
-                return FW_AUTO_INSTALLED
-            # Failed update (unknown reason)
+        # boot_type did not match (skip)
+        if boot_action != "cold":
+            return FW_AUTO_ERR_BOOT_TYPE
+
+        # Install firmware
+        if not self.install_firmware(image_path, allow_reboot=False):
             return FW_AUTO_ERR_UNKNOWN
 
-        # boot_type did not match (skip)
-        return FW_AUTO_ERR_BOOT_TYPE
+        # Installed pending next reboot
+        return FW_AUTO_INSTALLED
 
     @staticmethod
     def _read_generic_file(filename, len, ignore_errors=False):
@@ -443,13 +445,13 @@ class ComponentONIE(Component):
         self.description = self.COMPONENT_DESCRIPTION
         self.onie_updater = ONIEUpdater()
 
-    def __install_firmware(self, image_path):
+    def __install_firmware(self, image_path, allow_reboot=True):
         if not self._check_file_validity(image_path):
             return False
 
         try:
             print("INFO: Staging {} firmware update with ONIE updater".format(self.name))
-            self.onie_updater.update_firmware(image_path)
+            self.onie_updater.update_firmware(image_path, allow_reboot)
         except Exception as e:
             print("ERROR: Failed to update {} firmware: {}".format(self.name, str(e)))
             return False
@@ -469,8 +471,8 @@ class ComponentONIE(Component):
     def get_firmware_update_notification(self, image_path):
         return "Immediate cold reboot is required to complete {} firmware update".format(self.name)
 
-    def install_firmware(self, image_path):
-        return self.__install_firmware(image_path)
+    def install_firmware(self, image_path, allow_reboot=True):
+        return self.__install_firmware(image_path, allow_reboot)
 
     def update_firmware(self, image_path):
         self.__install_firmware(image_path)
@@ -488,6 +490,7 @@ class ComponentSSD(Component):
 
     SSD_INFO_COMMAND = "/usr/bin/mlnx-ssd-fw-update.sh -q"
     SSD_FIRMWARE_INFO_COMMAND = "/usr/bin/mlnx-ssd-fw-update.sh -q -i {}"
+    SSD_FIRMWARE_INSTALL_COMMAND = "/usr/bin/mlnx-ssd-fw-update.sh --no-power-cycle -y -u -i {}"
     SSD_FIRMWARE_UPDATE_COMMAND = "/usr/bin/mlnx-ssd-fw-update.sh -y -u -i {}"
 
     def __init__(self):
@@ -497,11 +500,14 @@ class ComponentSSD(Component):
         self.description = self.COMPONENT_DESCRIPTION
         self.image_ext_name = self.COMPONENT_FIRMWARE_EXTENSION
 
-    def __install_firmware(self, image_path):
+    def __install_firmware(self, image_path, allow_reboot=True):
         if not self._check_file_validity(image_path):
             return False
 
-        cmd = self.SSD_FIRMWARE_UPDATE_COMMAND.format(image_path)
+        if allow_reboot:
+            cmd = self.SSD_FIRMWARE_UPDATE_COMMAND.format(image_path)
+        else:
+            cmd = self.SSD_FIRMWARE_INSTALL_COMMAND.format(image_path)
 
         try:
             print("INFO: Installing {} firmware update".format(self.name))
@@ -519,9 +525,6 @@ class ComponentSSD(Component):
         then compares it against boot_action to determine whether to proceed with install.
         """
 
-        # All devices support cold boot
-        supported_boot = ['cold']
-
         # Verify image path exists
         if not os.path.exists(image_path):
             # Invalid image path
@@ -529,22 +532,21 @@ class ComponentSSD(Component):
 
         # Check if post_install reboot is required
         try:
-            if self.get_firmware_update_notification(image_path) is None:
-                # No power cycle required
-                supported_boot += ['warm', 'fast', 'none', 'any']
-        except RuntimeError:
-            # Unknown error from firmware probe
-            return FW_AUTO_ERR_UNKNOWN
-
-        if boot_action in supported_boot:
-            if self.update_firmware(image_path):
-                # Successful update
-                return FW_AUTO_INSTALLED
-            # Failed update (unknown reason)
-            return FW_AUTO_ERR_UNKNOWN
+            reboot_required = self.get_firmware_update_notification(image_path) is not None
+        except RuntimeError as e:
+            return FW_AUTO_ERR_UNKNOWN                    
+        
+        # Update if no reboot needed
+        if not reboot_required:
+            self.update_firmware(image_path)
+            return FW_AUTO_UPDATED
 
         # boot_type did not match (skip)
-        return FW_AUTO_ERR_BOOT_TYPE
+        if boot_action != "cold":
+            return FW_AUTO_ERR_BOOT_TYPE
+
+        # Schedule if we need a cold boot
+        return FW_AUTO_SCHEDULED
 
     def get_firmware_version(self):
         cmd = self.SSD_INFO_COMMAND
@@ -632,8 +634,8 @@ class ComponentSSD(Component):
 
         return notification
 
-    def install_firmware(self, image_path):
-        return self.__install_firmware(image_path)
+    def install_firmware(self, image_path, allow_reboot=True):
+        return self.__install_firmware(image_path, allow_reboot)
 
     def update_firmware(self, image_path):
         self.__install_firmware(image_path)
@@ -654,7 +656,7 @@ class ComponentBIOS(Component):
         self.image_ext_name = self.COMPONENT_FIRMWARE_EXTENSION
         self.onie_updater = ONIEUpdater()
 
-    def __install_firmware(self, image_path):
+    def __install_firmware(self, image_path, allow_reboot=True):
         if not self.onie_updater.is_non_onie_firmware_update_supported():
             print("ERROR: ONIE {} or later is required".format(self.onie_updater.get_onie_required_version()))
             return False
@@ -664,7 +666,7 @@ class ComponentBIOS(Component):
 
         try:
             print("INFO: Staging {} firmware update with ONIE updater".format(self.name))
-            self.onie_updater.update_firmware(image_path)
+            self.onie_updater.update_firmware(image_path, allow_reboot)
         except Exception as e:
             print("ERROR: Failed to update {} firmware: {}".format(self.name, str(e)))
             return False
@@ -689,8 +691,8 @@ class ComponentBIOS(Component):
     def get_firmware_update_notification(self, image_path):
         return "Immediate cold reboot is required to complete {} firmware update".format(self.name)
 
-    def install_firmware(self, image_path):
-        return self.__install_firmware(image_path)
+    def install_firmware(self, image_path, allow_reboot=True):
+        return self.__install_firmware(image_path, allow_reboot)
 
     def update_firmware(self, image_path):
         self.__install_firmware(image_path)
@@ -761,6 +763,29 @@ class ComponentCPLD(Component):
 
         return True
 
+    def auto_update_firmware(self, image_path, boot_action):
+        """
+        Default handling of attempted automatic update for a component of a Mellanox switch.
+        Will skip the installation if the boot_action is 'warm' or 'fast' and will call update_firmware()
+        if boot_action is fast.
+        """
+
+        # Verify image path exists
+        if not os.path.exists(image_path):
+            # Invalid image path
+            return FW_AUTO_ERR_IMAGE
+
+        # boot_type did not match (skip)
+        if boot_action != "cold":
+            return FW_AUTO_ERR_BOOT_TYPE
+
+        # Install burn. Error if fail.
+        if not self.install_firmware(image_path):
+            return FW_AUTO_ERR_UNKNOWN
+            
+        # Schedule refresh
+        return FW_AUTO_SCHEDULED    
+
     def get_firmware_version(self):
         part_number_file = self.CPLD_PART_NUMBER_FILE.format(self.idx)
         version_file = self.CPLD_VERSION_FILE.format(self.idx)
@@ -797,7 +822,17 @@ class ComponentCPLD(Component):
         return "Immediate power cycle is required to complete {} firmware update".format(self.name)
 
     def install_firmware(self, image_path):
-        return self.__install_firmware(image_path)
+        if MPFAManager.MPFA_EXTENSION in image_path:
+            with MPFAManager(image_path) as mpfa:
+                if not mpfa.get_metadata().has_option('firmware', 'burn'):
+                    raise RuntimeError("Failed to get {} burn firmware".format(self.name))
+
+                burn_firmware = mpfa.get_metadata().get('firmware', 'burn')
+
+                print("INFO: Processing {} burn file: firmware install".format(self.name))
+                return self.__install_firmware(os.path.join(mpfa.get_path(), burn_firmware))
+        else:
+            return self.__install_firmware(image_path)
 
     def update_firmware(self, image_path):
         with MPFAManager(image_path) as mpfa:

platform/mellanox/mlnx-platform-api/tests/test_firmware.py

@@ -24,23 +24,21 @@ test_path = os.path.dirname(os.path.abspath(__file__))
 modules_path = os.path.dirname(test_path)
 sys.path.insert(0, modules_path)
 
-from sonic_platform.component import Component, ComponentSSD
+from sonic_platform.component import Component, ComponentSSD, ComponentCPLD
 
 from sonic_platform_base.component_base import ComponentBase,           \
                                                 FW_AUTO_INSTALLED,      \
+                                                FW_AUTO_SCHEDULED,      \
+                                                FW_AUTO_UPDATED,        \
                                                 FW_AUTO_ERR_BOOT_TYPE,  \
-                                                FW_AUTO_ERR_IMAGE
-# Temp workaround to fix build issue, shall be refactor once sonic-platform-common submodule pointer is updated
-try:
-    from sonic_platform_base.component_base import FW_AUTO_ERR_UNKNOWN
-except ImportError as e:
-    from sonic_platform_base.component_base import FW_AUTO_ERR_UKNOWN as FW_AUTO_ERR_UNKNOWN
+                                                FW_AUTO_ERR_IMAGE,      \
+                                                FW_AUTO_ERR_UNKNOWN
 
 
-def mock_update_firmware_success(image_path):
+def mock_update_firmware_success(image_path, allow_reboot=False):
     return True
 
-def mock_update_firmware_fail(image_path):
+def mock_update_firmware_fail(image_path, allow_reboot=False):
     return False
 
 def mock_update_notification_cold_boot(image_path):
@@ -59,14 +57,20 @@ test_data_default = [
         (mock_update_firmware_success, True, 'cold', FW_AUTO_INSTALLED)
         ]
 
+test_data_cpld = [
+        (None, False, None, FW_AUTO_ERR_IMAGE),
+        (None, True, 'warm', FW_AUTO_ERR_BOOT_TYPE),
+        (mock_update_firmware_fail, True, 'cold', FW_AUTO_ERR_UNKNOWN),
+        (mock_update_firmware_success, True, 'cold', FW_AUTO_SCHEDULED)
+        ]
+
 test_data_ssd = [
         (None, None, False, None, FW_AUTO_ERR_IMAGE),
         (None, mock_update_notification_error, True, None, FW_AUTO_ERR_UNKNOWN),
-        (mock_update_firmware_fail,    mock_update_notification_cold_boot, True, 'cold', FW_AUTO_ERR_UNKNOWN),
         (mock_update_firmware_success, mock_update_notification_cold_boot, True, 'warm', FW_AUTO_ERR_BOOT_TYPE),
-        (mock_update_firmware_success, mock_update_notification_cold_boot, True, 'cold', FW_AUTO_INSTALLED),
-        (mock_update_firmware_success, mock_update_notification_warm_boot, True, 'warm', FW_AUTO_INSTALLED),
-        (mock_update_firmware_success, mock_update_notification_warm_boot, True, 'cold', FW_AUTO_INSTALLED)
+        (mock_update_firmware_success, mock_update_notification_cold_boot, True, 'cold', FW_AUTO_SCHEDULED),
+        (mock_update_firmware_success, mock_update_notification_warm_boot, True, 'warm', FW_AUTO_UPDATED),
+        (mock_update_firmware_success, mock_update_notification_warm_boot, True, 'cold', FW_AUTO_UPDATED)
         ]
 
 @pytest.mark.parametrize('update_func, image_found, boot_type, expect', test_data_default)
@@ -77,7 +81,23 @@ def test_auto_update_firmware_default(monkeypatch, update_func, image_found, boo
 
     test_component = Component()
 
-    monkeypatch.setattr(test_component, 'update_firmware', update_func)
+    monkeypatch.setattr(test_component, 'install_firmware', update_func)
+    monkeypatch.setattr(os.path, 'exists', mock_path_exists)
+
+    result = test_component.auto_update_firmware(None, boot_type)
+
+    assert result == expect
+
+
+@pytest.mark.parametrize('update_func, image_found, boot_type, expect', test_data_cpld)
+def test_auto_update_firmware_cpld(monkeypatch, update_func, image_found, boot_type, expect):
+
+    def mock_path_exists(path):
+        return image_found
+
+    test_component = ComponentCPLD(0)
+
+    monkeypatch.setattr(test_component, 'install_firmware', update_func)
     monkeypatch.setattr(os.path, 'exists', mock_path_exists)
 
     result = test_component.auto_update_firmware(None, boot_type)
@@ -86,7 +106,7 @@ def test_auto_update_firmware_default(monkeypatch, update_func, image_found, boo
 
 
 @pytest.mark.parametrize('update_func, notify, image_found, boot_type, expect', test_data_ssd)
-def test_auto_update_firmware_default(monkeypatch, update_func, notify, image_found, boot_type, expect):
+def test_auto_update_firmware_ssd(monkeypatch, update_func, notify, image_found, boot_type, expect):
 
     def mock_path_exists(path):
         return image_found

platform/mellanox/mlnx-ssd-fw-update.sh

@@ -48,6 +48,7 @@ ARG_IMAGE_VAL=""
 ARG_QUERY_FLAG=$FALSE
 ARG_YES_FLAG=$FALSE
 ARG_POWER_CYCLE_FLAG=$FALSE
+ARG_FORCE_POWER_CYCLE_FLAG=$FALSE
 ARG_HELP_FLAG=$FALSE
 ARG_VERSION_FLAG=$FALSE
 ARG_PACKAGE_INFO_FLAG=$FALSE
@@ -178,6 +179,10 @@ function check_usage() {
             ARG_POWER_CYCLE_FLAG=$TRUE
             shift # past argument
             ;;
+        --no-power-cycle)
+            ARG_FORCE_NO_POWER_CYCLE_FLAG=$TRUE
+            shift # past argument
+            ;;
         *)
             LOG_MSG "Error: false usage given."
             usage
@@ -197,6 +202,7 @@ function check_usage() {
           ("$ARG_UPDATE_FLAG" == "$TRUE" && "$ARG_IMAGE_FLAG" == "$FALSE") ||
           ("$ARG_PACKAGE_INFO_FLAG" == "$TRUE" && "$ARG_IMAGE_FLAG" == "$FALSE") ||
           ("$ARG_POWER_CYCLE_FLAG" == "$TRUE" && "$ARG_UPDATE_FLAG" == "$FALSE") ||
+          ("$ARG_FORCE_NO_POWER_CYCLE_FLAG" == "$TRUE" && "$ARG_POWER_CYCLE_FLAG" == "$TRUE") ||
           ("$ARG_UPDATE_FLAG" == "$TRUE" && "$ARG_PACKAGE_INFO_FLAG" == "$TRUE") ]]; then
 
         LOG_MSG "Error: false usage given."
@@ -213,6 +219,7 @@ function check_usage() {
     LOG_MSG "ARG_VERSION_FLAG          = ${ARG_VERSION_FLAG}"           ${DEBUG_MSG}
     LOG_MSG "ARG_PACKAGE_INFO_FLAG     = ${ARG_PACKAGE_INFO_FLAG}"      ${DEBUG_MSG}
     LOG_MSG "ARG_POWER_CYCLE_FLAG      = ${ARG_POWER_CYCLE_FLAG}"       ${DEBUG_MSG}
+    LOG_MSG "ARG_FORCE_NO_POWER_CYCLE_FLAG      = ${ARG_FORCE_NO_POWER_CYCLE_FLAG}"       ${DEBUG_MSG}
 
 }
 
@@ -280,12 +287,12 @@ function get_ssd_info() {
 #=
 function check_tool_dependencies() {
     LOG_MSG "func: ${FUNCNAME[0]}()" ${DEBUG_MSG}
-   	for i in "${!DEPENDECIES[@]}"
-	do
-		if [ ! -x "$(command -v ${DEPENDECIES[$i]})" ]; then
-			LOG_MSG_AND_EXIT "Error: This tool require the following utils to be installed ${DEPENDECIES[$i]}"
-		fi
-	done
+    for i in "${!DEPENDECIES[@]}"
+    do
+        if [ ! -x "$(command -v ${DEPENDECIES[$i]})" ]; then
+            LOG_MSG_AND_EXIT "Error: This tool require the following utils to be installed ${DEPENDECIES[$i]}"
+        fi
+    done
 }
 
 #==============================================================================#
@@ -673,7 +680,12 @@ elif [ $ARG_UPDATE_FLAG == $TRUE ]; then
                 if [ ! -f $ssd_script_path ]; then
                     LOG_MSG_AND_EXIT "Error: fail to call upgrade script ($ssd_script_path)!"
                 fi
-                ( 
+                (
+                    if [[ "yes" == "$power_policy" && $ARG_FORCE_NO_POWER_CYCLE_FLAG == $TRUE ]]; then
+                        # If a power cycle is required and we are not power cycling automatically lock the file system for safety
+                        LOG_MSG "Immediate power cycle is required but override flag has been given. Locking file system as read only to protect system integrity."
+                        echo u > /proc/sysrq-trigger
+                    fi
                     cd "${extraction_path}/${section}" > /dev/null 2>&1 || exit
                     /bin/bash "$ssd_script_path" "${extraction_path}/${section}"
                     #cd - > /dev/null 2>&1 || exit
@@ -684,6 +696,11 @@ elif [ $ARG_UPDATE_FLAG == $TRUE ]; then
                     LOG_MSG "SSD FW update completed successfully."
 
                     if [[ "yes" == "$power_policy" || $ARG_POWER_CYCLE_FLAG == $TRUE ]]; then
+
+                        if [[ $ARG_FORCE_NO_POWER_CYCLE_FLAG == $TRUE ]]; then
+                            LOG_MSG_AND_EXIT "An IMMEDIATE power cycle is REQUIRED to upgrade the SSD. Please perform a cold reboot as soon as possible."
+                        fi
+                        
                         LOG_MSG "Execute power cycle..."
                         sleep 1
                         sync

platform/mellanox/rules.dep

@@ -15,3 +15,4 @@ include $(PLATFORM_PATH)/mlnx-ffb.dep
 include $(PLATFORM_PATH)/issu-version.dep
 include $(PLATFORM_PATH)/mlnx-onie-fw-update.dep
 include $(PLATFORM_PATH)/mlnx-ssd-fw-update.dep
+include $(PLATFORM_PATH)/install-pending-fw.dep

platform/mellanox/rules.mk

@@ -29,6 +29,7 @@ include $(PLATFORM_PATH)/mlnx-ffb.mk
 include $(PLATFORM_PATH)/issu-version.mk
 include $(PLATFORM_PATH)/mlnx-onie-fw-update.mk
 include $(PLATFORM_PATH)/mlnx-ssd-fw-update.mk
+include $(PLATFORM_PATH)/install-pending-fw.mk
 
 SONIC_ALL += $(SONIC_ONE_IMAGE) \
              $(DOCKER_FPM)