sonic-buildimage/platform/mellanox/mlnx-fw-upgrade.j2
DavidZagury 4cc84c68dc
[Mellanox] Improve FW upgrade logging (#13465)
- Why I did it
To improve ASIC FW upgrade logging and have information about the cause of FW update failure in the log.

- How I did it
Added syslog logger support

In case the FW update has failed the update tool will give the cause of the failure in the output in the last line, starting with "Fail".
When running the tool, in case of a failed update, we will parse the output to retrieve the cause and log it.

Device #1:
 ----------
 
 Device Type:      ConnectX6DX
   Part Number:      MCX623106AN-CDA_Ax
   Description:      ConnectX-6 Dx EN adapter card; 100GbE; Dual-port QSFP56; PCIe 4.0/3.0 x16;
   PSID:             MT_0000000359
   PCI Device Name:  /dev/mst/mt4125_pciconf0
   Base GUID:        0c42a103007d22d4
   Base MAC:         0c42a17d22d4
   Versions:         Current        Available     
      FW             22.32.0498     22.32.0498    
      PXE            3.6.0500       3.6.0500      
      UEFI           14.25.0015     14.25.0015    
 
 Status:           Forced update required
 
---------
 Found 1 device(s) requiring firmware update...
 
Device #1: Updating FW ...     
 FSMST_INITIALIZE -   OK          
 Writing Boot image component -   OK          
 Fail : The Digest in the signature is wrong

- How to verify it
mlnx-fw-upgrade.sh --upgrade
2023-01-25 20:53:39 +02:00

318 lines
8.3 KiB
Django/Jinja
Executable File

#!/bin/bash
declare -r SCRIPT_NAME="$(basename "$0")"
declare -r SCRIPT_PATH="$(readlink -f "$0")"
declare -r SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
declare -r YES_PARAM="yes"
declare -r NO_PARAM="no"
declare -r VERBOSE_ERROR="1"
declare -r VERBOSE_WARNING="2"
declare -r VERBOSE_NOTICE="3"
declare -r VERBOSE_INFO="4"
declare -r VERBOSE_MAX="${VERBOSE_INFO}"
declare -r VERBOSE_MIN="${VERBOSE_ERROR}"
declare -r EXIT_SUCCESS="0"
declare -r EXIT_FAILURE="1"
declare -r QUERY_CMD="mlxfwmanager --query"
declare -r LIST_CONTENT_CMD="mlxfwmanager --list-content"
declare -r BURN_CMD="mlxfwmanager -u -f -y"
declare -r QUERY_FILE="/tmp/mlxfwmanager-query.log"
declare -r LIST_CONTENT_FILE="/tmp/mlxfwmanager-list-content.log"
declare -r SPC1_ASIC="spc1"
declare -r SPC2_ASIC="spc2"
declare -r SPC3_ASIC="spc3"
declare -r SPC4_ASIC="spc4"
declare -r UNKN_ASIC="unknown"
declare -r UNKN_MST="unknown"
declare -rA FW_FILE_MAP=( \
[$SPC1_ASIC]="/etc/mlnx/fw-SPC.mfa" \
[$SPC2_ASIC]="/etc/mlnx/fw-SPC2.mfa" \
[$SPC3_ASIC]="/etc/mlnx/fw-SPC3.mfa" \
[$SPC4_ASIC]="/etc/mlnx/fw-SPC4.mfa" \
)
IMAGE_UPGRADE="${NO_PARAM}"
SYSLOG_LOGGER="${NO_PARAM}"
VERBOSE_LEVEL="${VERBOSE_MIN}"
function PrintHelp() {
echo
echo "Usage: ./${SCRIPT_NAME} [OPTIONS]"
echo
echo "OPTIONS:"
echo " -u, --upgrade Upgrade ASIC firmware using next boot image (useful after SONiC-To-SONiC update)"
echo " -s, --syslog Use syslog logger (enabled when -u|--upgrade)"
echo " -v, --verbose Verbose mode (enabled when -u|--upgrade)"
echo " -h, --help Print help"
echo
echo "Examples:"
echo " ./${SCRIPT_NAME} --verbose"
echo " ./${SCRIPT_NAME} --upgrade"
echo " ./${SCRIPT_NAME} --help"
echo
}
function ParseArguments() {
while [ "$#" -ge "1" ]; do
case "$1" in
-u|--upgrade)
IMAGE_UPGRADE="${YES_PARAM}"
SYSLOG_LOGGER="${YES_PARAM}"
;;
-v|--verbose)
VERBOSE_LEVEL="${VERBOSE_MAX}"
;;
-s|--syslog)
SYSLOG_LOGGER="${YES_PARAM}"
;;
-h|--help)
PrintHelp
exit "${EXIT_SUCCESS}"
;;
esac
shift
done
}
function LogError() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_ERROR}" ]]; then
echo "ERROR: $*"
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
fi
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
fi
}
function LogWarning() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_WARNING}" ]]; then
echo "WARNING: $*"
fi
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "WARNING" -t "${SCRIPT_NAME}" "$*"
fi
}
function LogNotice() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_NOTICE}" ]]; then
echo "NOTICE: $*"
fi
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "NOTICE" -t "${SCRIPT_NAME}" "$*"
fi
}
function LogInfo() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_INFO}" ]]; then
echo "INFO: $*"
fi
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "INFO" -t "${SCRIPT_NAME}" "$*"
fi
}
function ExitFailure() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_ERROR}" ]]; then
echo
LogError "$@"
echo
fi
exit "${EXIT_FAILURE}"
}
function ExitSuccess() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_INFO}" ]]; then
echo
LogInfo "$@"
echo
fi
exit "${EXIT_SUCCESS}"
}
function WaitForDevice() {
local -i QUERY_RETRY_COUNT_MAX="10"
local -i QUERY_RETRY_COUNT="0"
${QUERY_CMD} > /dev/null
while [[ ("${QUERY_RETRY_COUNT}" -lt "${QUERY_RETRY_COUNT_MAX}") && ("$?" -ne "${EXIT_SUCCESS}") ]]; do
sleep 1s
((QUERY_RETRY_COUNT++))
${QUERY_CMD} > /dev/null
done
}
function GetAsicType() {
local -r VENDOR_ID="15b3"
local -r SPC1_PRODUCT_ID="cb84"
local -r SPC2_PRODUCT_ID="cf6c"
local -r SPC3_PRODUCT_ID="cf70"
local -r SPC4_PRODUCT_ID="cf80"
if lspci -n | grep "${VENDOR_ID}:${SPC1_PRODUCT_ID}" &>/dev/null; then
echo "${SPC1_ASIC}"
exit "${EXIT_SUCCESS}"
elif lspci -n | grep "${VENDOR_ID}:${SPC2_PRODUCT_ID}" &>/dev/null; then
echo "${SPC2_ASIC}"
exit "${EXIT_SUCCESS}"
elif lspci -n | grep "${VENDOR_ID}:${SPC3_PRODUCT_ID}" &>/dev/null; then
echo "${SPC3_ASIC}"
exit "${EXIT_SUCCESS}"
elif lspci -n | grep "${VENDOR_ID}:${SPC4_PRODUCT_ID}" &>/dev/null; then
echo "${SPC4_ASIC}"
exit "${EXIT_SUCCESS}"
fi
echo "${UNKN_ASIC}"
exit "${EXIT_FAILURE}"
}
function GetMstDevice() {
local _MST_DEVICE="$(ls /dev/mst/*_pci_cr0 2>&1)"
if [[ ! -c "${_MST_DEVICE}" ]]; then
echo "${UNKN_MST}"
else
echo "${_MST_DEVICE}"
fi
exit "${EXIT_SUCCESS}"
}
function RunCmd() {
local ERROR_CODE="${EXIT_SUCCESS}"
if [[ "${VERBOSE_LEVEL}" -eq "${VERBOSE_MAX}" ]]; then
eval "$@"
else
eval "$@" &>/dev/null
fi
ERROR_CODE="$?"
if [[ "${ERROR_CODE}" != "${EXIT_SUCCESS}" ]]; then
ExitFailure "command failed: $@"
fi
}
function RunFwUpdateCmd() {
local ERROR_CODE="${EXIT_SUCCESS}"
local COMMAND="${BURN_CMD} $@"
if [[ "${VERBOSE_LEVEL}" -eq "${VERBOSE_MAX}" ]]; then
output=$(eval "${COMMAND}")
else
output=$(eval "${COMMAND}") >/dev/null 2>&1
fi
ERROR_CODE="$?"
if [[ "${ERROR_CODE}" != "${EXIT_SUCCESS}" ]]; then
failure_msg="${output#*Fail : }"
ExitFailure "FW Update command: ${COMMAND} failed with error: ${failure_msg}"
fi
}
function UpgradeFW() {
local -r _FS_MOUNTPOINT="$1"
local -r _ASIC_TYPE="$(GetAsicType)"
if [[ "${_ASIC_TYPE}" = "${UNKN_ASIC}" ]]; then
ExitFailure "failed to detect ASIC type"
fi
if [ ! -z "${_FS_MOUNTPOINT}" ]; then
local -r _FW_FILE="${_FS_MOUNTPOINT}/${FW_FILE_MAP[$_ASIC_TYPE]}"
else
local -r _FW_FILE="${FW_FILE_MAP[$_ASIC_TYPE]}"
fi
if [ ! -f "${_FW_FILE}" ]; then
ExitFailure "no such file: ${_FW_FILE}"
fi
RunCmd "${QUERY_CMD} -o ${QUERY_FILE}"
local -r _FW_CURRENT_INFO="$(grep FW ${QUERY_FILE})"
local -r _FW_CURRENT="$(echo ${_FW_CURRENT_INFO} | cut -f2 -d' ')"
local -r _PSID_INFO="$(grep PSID ${QUERY_FILE})"
local -r _PSID="$(echo ${_PSID_INFO} | cut -f2 -d' ')"
RunCmd "${LIST_CONTENT_CMD} -i ${_FW_FILE} -o ${LIST_CONTENT_FILE}"
local -r _FW_AVAILABLE_INFO="$(grep ${_PSID} ${LIST_CONTENT_FILE})"
local -r _FW_AVAILABLE="$(echo ${_FW_AVAILABLE_INFO} | cut -f4 -d' ')"
if [[ -z "${_FW_CURRENT}" ]]; then
ExitFailure "could not retreive current FW version"
fi
if [[ -z "${_FW_AVAILABLE}" ]]; then
ExitFailure "could not retreive available FW version"
fi
if [[ "${_FW_CURRENT}" == "${_FW_AVAILABLE}" ]]; then
ExitSuccess "firmware is up to date"
else
LogNotice "firmware upgrade is required. Installing compatible version..."
local -r _MST_DEVICE="$(GetMstDevice)"
if [[ "${_MST_DEVICE}" = "${UNKN_MST}" ]]; then
LogWarning "could not find fastest mst device, using default device"
RunFwUpdateCmd "-i ${_FW_FILE}"
else
RunFwUpdateCmd "-d ${_MST_DEVICE} -i ${_FW_FILE}"
fi
fi
}
function UpgradeFWFromImage() {
local -r _NEXT_SONIC_IMAGE="$(sonic-installer list | grep "Next: " | cut -f2 -d' ')"
local -r _CURRENT_SONIC_IMAGE="$(sonic-installer list | grep "Current: " | cut -f2 -d' ')"
local -r _FS_PATH="/host/image-${_NEXT_SONIC_IMAGE#SONiC-OS-}/fs.squashfs"
local -r _FS_MOUNTPOINT="/tmp/image-${_NEXT_SONIC_IMAGE#SONiC-OS-}-fs"
if [[ "${_CURRENT_SONIC_IMAGE}" == "${_NEXT_SONIC_IMAGE}" ]]; then
ExitSuccess "firmware is up to date"
else
mkdir -p "${_FS_MOUNTPOINT}"
mount -t squashfs "${_FS_PATH}" "${_FS_MOUNTPOINT}"
UpgradeFW "${_FS_MOUNTPOINT}"
umount -rf "${_FS_MOUNTPOINT}"
rm -rf "${_FS_MOUNTPOINT}"
fi
}
function ExitIfQEMU() {
if [ -n "$(lspci -vvv | grep SimX)" ]; then
ExitSuccess "No FW upgrade for SimX platform"
fi
}
ParseArguments "$@"
ExitIfQEMU
WaitForDevice
if [ "${IMAGE_UPGRADE}" != "${YES_PARAM}" ]; then
UpgradeFW
else
UpgradeFWFromImage
fi
ExitSuccess "firmware upgrade is completed"