4cc84c68dc
- Why I did it To improve ASIC FW upgrade logging and have information about the cause of FW update failure in the log. - How I did it Added syslog logger support In case the FW update has failed the update tool will give the cause of the failure in the output in the last line, starting with "Fail". When running the tool, in case of a failed update, we will parse the output to retrieve the cause and log it. Device #1: ---------- Device Type: ConnectX6DX Part Number: MCX623106AN-CDA_Ax Description: ConnectX-6 Dx EN adapter card; 100GbE; Dual-port QSFP56; PCIe 4.0/3.0 x16; PSID: MT_0000000359 PCI Device Name: /dev/mst/mt4125_pciconf0 Base GUID: 0c42a103007d22d4 Base MAC: 0c42a17d22d4 Versions: Current Available FW 22.32.0498 22.32.0498 PXE 3.6.0500 3.6.0500 UEFI 14.25.0015 14.25.0015 Status: Forced update required --------- Found 1 device(s) requiring firmware update... Device #1: Updating FW ... FSMST_INITIALIZE - OK Writing Boot image component - OK Fail : The Digest in the signature is wrong - How to verify it mlnx-fw-upgrade.sh --upgrade
318 lines
8.3 KiB
Django/Jinja
Executable File
318 lines
8.3 KiB
Django/Jinja
Executable File
#!/bin/bash
|
|
|
|
declare -r SCRIPT_NAME="$(basename "$0")"
|
|
declare -r SCRIPT_PATH="$(readlink -f "$0")"
|
|
declare -r SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
|
|
|
|
declare -r YES_PARAM="yes"
|
|
declare -r NO_PARAM="no"
|
|
|
|
declare -r VERBOSE_ERROR="1"
|
|
declare -r VERBOSE_WARNING="2"
|
|
declare -r VERBOSE_NOTICE="3"
|
|
declare -r VERBOSE_INFO="4"
|
|
|
|
declare -r VERBOSE_MAX="${VERBOSE_INFO}"
|
|
declare -r VERBOSE_MIN="${VERBOSE_ERROR}"
|
|
|
|
declare -r EXIT_SUCCESS="0"
|
|
declare -r EXIT_FAILURE="1"
|
|
|
|
declare -r QUERY_CMD="mlxfwmanager --query"
|
|
declare -r LIST_CONTENT_CMD="mlxfwmanager --list-content"
|
|
declare -r BURN_CMD="mlxfwmanager -u -f -y"
|
|
|
|
declare -r QUERY_FILE="/tmp/mlxfwmanager-query.log"
|
|
declare -r LIST_CONTENT_FILE="/tmp/mlxfwmanager-list-content.log"
|
|
|
|
declare -r SPC1_ASIC="spc1"
|
|
declare -r SPC2_ASIC="spc2"
|
|
declare -r SPC3_ASIC="spc3"
|
|
declare -r SPC4_ASIC="spc4"
|
|
declare -r UNKN_ASIC="unknown"
|
|
declare -r UNKN_MST="unknown"
|
|
|
|
declare -rA FW_FILE_MAP=( \
|
|
[$SPC1_ASIC]="/etc/mlnx/fw-SPC.mfa" \
|
|
[$SPC2_ASIC]="/etc/mlnx/fw-SPC2.mfa" \
|
|
[$SPC3_ASIC]="/etc/mlnx/fw-SPC3.mfa" \
|
|
[$SPC4_ASIC]="/etc/mlnx/fw-SPC4.mfa" \
|
|
)
|
|
|
|
IMAGE_UPGRADE="${NO_PARAM}"
|
|
SYSLOG_LOGGER="${NO_PARAM}"
|
|
VERBOSE_LEVEL="${VERBOSE_MIN}"
|
|
|
|
function PrintHelp() {
|
|
echo
|
|
echo "Usage: ./${SCRIPT_NAME} [OPTIONS]"
|
|
echo
|
|
echo "OPTIONS:"
|
|
echo " -u, --upgrade Upgrade ASIC firmware using next boot image (useful after SONiC-To-SONiC update)"
|
|
echo " -s, --syslog Use syslog logger (enabled when -u|--upgrade)"
|
|
echo " -v, --verbose Verbose mode (enabled when -u|--upgrade)"
|
|
echo " -h, --help Print help"
|
|
echo
|
|
echo "Examples:"
|
|
echo " ./${SCRIPT_NAME} --verbose"
|
|
echo " ./${SCRIPT_NAME} --upgrade"
|
|
echo " ./${SCRIPT_NAME} --help"
|
|
echo
|
|
}
|
|
|
|
function ParseArguments() {
|
|
while [ "$#" -ge "1" ]; do
|
|
case "$1" in
|
|
-u|--upgrade)
|
|
IMAGE_UPGRADE="${YES_PARAM}"
|
|
SYSLOG_LOGGER="${YES_PARAM}"
|
|
;;
|
|
-v|--verbose)
|
|
VERBOSE_LEVEL="${VERBOSE_MAX}"
|
|
;;
|
|
-s|--syslog)
|
|
SYSLOG_LOGGER="${YES_PARAM}"
|
|
;;
|
|
-h|--help)
|
|
PrintHelp
|
|
exit "${EXIT_SUCCESS}"
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
}
|
|
|
|
function LogError() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_ERROR}" ]]; then
|
|
echo "ERROR: $*"
|
|
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
|
|
fi
|
|
|
|
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
|
|
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
|
|
fi
|
|
}
|
|
|
|
function LogWarning() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_WARNING}" ]]; then
|
|
echo "WARNING: $*"
|
|
fi
|
|
|
|
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
|
|
logger -p "WARNING" -t "${SCRIPT_NAME}" "$*"
|
|
fi
|
|
}
|
|
|
|
function LogNotice() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_NOTICE}" ]]; then
|
|
echo "NOTICE: $*"
|
|
fi
|
|
|
|
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
|
|
logger -p "NOTICE" -t "${SCRIPT_NAME}" "$*"
|
|
fi
|
|
}
|
|
|
|
function LogInfo() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_INFO}" ]]; then
|
|
echo "INFO: $*"
|
|
fi
|
|
|
|
if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
|
|
logger -p "INFO" -t "${SCRIPT_NAME}" "$*"
|
|
fi
|
|
}
|
|
|
|
function ExitFailure() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_ERROR}" ]]; then
|
|
echo
|
|
LogError "$@"
|
|
echo
|
|
fi
|
|
|
|
exit "${EXIT_FAILURE}"
|
|
}
|
|
|
|
function ExitSuccess() {
|
|
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_INFO}" ]]; then
|
|
echo
|
|
LogInfo "$@"
|
|
echo
|
|
fi
|
|
|
|
exit "${EXIT_SUCCESS}"
|
|
}
|
|
|
|
function WaitForDevice() {
|
|
local -i QUERY_RETRY_COUNT_MAX="10"
|
|
local -i QUERY_RETRY_COUNT="0"
|
|
|
|
${QUERY_CMD} > /dev/null
|
|
|
|
while [[ ("${QUERY_RETRY_COUNT}" -lt "${QUERY_RETRY_COUNT_MAX}") && ("$?" -ne "${EXIT_SUCCESS}") ]]; do
|
|
sleep 1s
|
|
((QUERY_RETRY_COUNT++))
|
|
${QUERY_CMD} > /dev/null
|
|
done
|
|
}
|
|
|
|
function GetAsicType() {
|
|
local -r VENDOR_ID="15b3"
|
|
|
|
local -r SPC1_PRODUCT_ID="cb84"
|
|
local -r SPC2_PRODUCT_ID="cf6c"
|
|
local -r SPC3_PRODUCT_ID="cf70"
|
|
local -r SPC4_PRODUCT_ID="cf80"
|
|
|
|
if lspci -n | grep "${VENDOR_ID}:${SPC1_PRODUCT_ID}" &>/dev/null; then
|
|
echo "${SPC1_ASIC}"
|
|
exit "${EXIT_SUCCESS}"
|
|
elif lspci -n | grep "${VENDOR_ID}:${SPC2_PRODUCT_ID}" &>/dev/null; then
|
|
echo "${SPC2_ASIC}"
|
|
exit "${EXIT_SUCCESS}"
|
|
elif lspci -n | grep "${VENDOR_ID}:${SPC3_PRODUCT_ID}" &>/dev/null; then
|
|
echo "${SPC3_ASIC}"
|
|
exit "${EXIT_SUCCESS}"
|
|
elif lspci -n | grep "${VENDOR_ID}:${SPC4_PRODUCT_ID}" &>/dev/null; then
|
|
echo "${SPC4_ASIC}"
|
|
exit "${EXIT_SUCCESS}"
|
|
fi
|
|
|
|
echo "${UNKN_ASIC}"
|
|
exit "${EXIT_FAILURE}"
|
|
}
|
|
|
|
function GetMstDevice() {
|
|
local _MST_DEVICE="$(ls /dev/mst/*_pci_cr0 2>&1)"
|
|
|
|
if [[ ! -c "${_MST_DEVICE}" ]]; then
|
|
echo "${UNKN_MST}"
|
|
else
|
|
echo "${_MST_DEVICE}"
|
|
fi
|
|
|
|
exit "${EXIT_SUCCESS}"
|
|
}
|
|
|
|
function RunCmd() {
|
|
local ERROR_CODE="${EXIT_SUCCESS}"
|
|
|
|
if [[ "${VERBOSE_LEVEL}" -eq "${VERBOSE_MAX}" ]]; then
|
|
eval "$@"
|
|
else
|
|
eval "$@" &>/dev/null
|
|
fi
|
|
|
|
ERROR_CODE="$?"
|
|
if [[ "${ERROR_CODE}" != "${EXIT_SUCCESS}" ]]; then
|
|
ExitFailure "command failed: $@"
|
|
fi
|
|
}
|
|
|
|
function RunFwUpdateCmd() {
|
|
local ERROR_CODE="${EXIT_SUCCESS}"
|
|
local COMMAND="${BURN_CMD} $@"
|
|
|
|
if [[ "${VERBOSE_LEVEL}" -eq "${VERBOSE_MAX}" ]]; then
|
|
output=$(eval "${COMMAND}")
|
|
else
|
|
output=$(eval "${COMMAND}") >/dev/null 2>&1
|
|
fi
|
|
|
|
ERROR_CODE="$?"
|
|
if [[ "${ERROR_CODE}" != "${EXIT_SUCCESS}" ]]; then
|
|
failure_msg="${output#*Fail : }"
|
|
ExitFailure "FW Update command: ${COMMAND} failed with error: ${failure_msg}"
|
|
fi
|
|
}
|
|
|
|
function UpgradeFW() {
|
|
local -r _FS_MOUNTPOINT="$1"
|
|
|
|
local -r _ASIC_TYPE="$(GetAsicType)"
|
|
if [[ "${_ASIC_TYPE}" = "${UNKN_ASIC}" ]]; then
|
|
ExitFailure "failed to detect ASIC type"
|
|
fi
|
|
|
|
if [ ! -z "${_FS_MOUNTPOINT}" ]; then
|
|
local -r _FW_FILE="${_FS_MOUNTPOINT}/${FW_FILE_MAP[$_ASIC_TYPE]}"
|
|
else
|
|
local -r _FW_FILE="${FW_FILE_MAP[$_ASIC_TYPE]}"
|
|
fi
|
|
|
|
if [ ! -f "${_FW_FILE}" ]; then
|
|
ExitFailure "no such file: ${_FW_FILE}"
|
|
fi
|
|
|
|
RunCmd "${QUERY_CMD} -o ${QUERY_FILE}"
|
|
local -r _FW_CURRENT_INFO="$(grep FW ${QUERY_FILE})"
|
|
local -r _FW_CURRENT="$(echo ${_FW_CURRENT_INFO} | cut -f2 -d' ')"
|
|
local -r _PSID_INFO="$(grep PSID ${QUERY_FILE})"
|
|
local -r _PSID="$(echo ${_PSID_INFO} | cut -f2 -d' ')"
|
|
|
|
RunCmd "${LIST_CONTENT_CMD} -i ${_FW_FILE} -o ${LIST_CONTENT_FILE}"
|
|
local -r _FW_AVAILABLE_INFO="$(grep ${_PSID} ${LIST_CONTENT_FILE})"
|
|
local -r _FW_AVAILABLE="$(echo ${_FW_AVAILABLE_INFO} | cut -f4 -d' ')"
|
|
|
|
if [[ -z "${_FW_CURRENT}" ]]; then
|
|
ExitFailure "could not retreive current FW version"
|
|
fi
|
|
|
|
if [[ -z "${_FW_AVAILABLE}" ]]; then
|
|
ExitFailure "could not retreive available FW version"
|
|
fi
|
|
|
|
if [[ "${_FW_CURRENT}" == "${_FW_AVAILABLE}" ]]; then
|
|
ExitSuccess "firmware is up to date"
|
|
else
|
|
LogNotice "firmware upgrade is required. Installing compatible version..."
|
|
local -r _MST_DEVICE="$(GetMstDevice)"
|
|
if [[ "${_MST_DEVICE}" = "${UNKN_MST}" ]]; then
|
|
LogWarning "could not find fastest mst device, using default device"
|
|
RunFwUpdateCmd "-i ${_FW_FILE}"
|
|
else
|
|
RunFwUpdateCmd "-d ${_MST_DEVICE} -i ${_FW_FILE}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function UpgradeFWFromImage() {
|
|
local -r _NEXT_SONIC_IMAGE="$(sonic-installer list | grep "Next: " | cut -f2 -d' ')"
|
|
local -r _CURRENT_SONIC_IMAGE="$(sonic-installer list | grep "Current: " | cut -f2 -d' ')"
|
|
|
|
local -r _FS_PATH="/host/image-${_NEXT_SONIC_IMAGE#SONiC-OS-}/fs.squashfs"
|
|
local -r _FS_MOUNTPOINT="/tmp/image-${_NEXT_SONIC_IMAGE#SONiC-OS-}-fs"
|
|
|
|
if [[ "${_CURRENT_SONIC_IMAGE}" == "${_NEXT_SONIC_IMAGE}" ]]; then
|
|
ExitSuccess "firmware is up to date"
|
|
else
|
|
mkdir -p "${_FS_MOUNTPOINT}"
|
|
mount -t squashfs "${_FS_PATH}" "${_FS_MOUNTPOINT}"
|
|
|
|
UpgradeFW "${_FS_MOUNTPOINT}"
|
|
|
|
umount -rf "${_FS_MOUNTPOINT}"
|
|
rm -rf "${_FS_MOUNTPOINT}"
|
|
fi
|
|
}
|
|
|
|
function ExitIfQEMU() {
|
|
if [ -n "$(lspci -vvv | grep SimX)" ]; then
|
|
ExitSuccess "No FW upgrade for SimX platform"
|
|
fi
|
|
}
|
|
|
|
ParseArguments "$@"
|
|
|
|
ExitIfQEMU
|
|
|
|
WaitForDevice
|
|
|
|
if [ "${IMAGE_UPGRADE}" != "${YES_PARAM}" ]; then
|
|
UpgradeFW
|
|
else
|
|
UpgradeFWFromImage
|
|
fi
|
|
|
|
ExitSuccess "firmware upgrade is completed"
|