sonic-buildimage/platform/broadcom/sonic-platform-modules-ragile/common/script/warm_upgrade.py

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import sys
import os
import time
import syslog
import signal
import click
from platform_util import get_value, set_value, exec_os_cmd, exec_os_cmd_log
from platform_config import WARM_UPGRADE_PARAM


WARM_UPGRADE_DEBUG_FILE = "/etc/.warm_upgrade_debug_flag"

WARMUPGRADEDEBUG = 1

debuglevel = 0

CONTEXT_SETTINGS = {"help_option_names": ['-h', '--help']}


class AliasedGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        rv = click.Group.get_command(self, ctx, cmd_name)
        if rv is not None:
            return rv
        matches = [x for x in self.list_commands(ctx)
                   if x.startswith(cmd_name)]
        if not matches:
            return None
        if len(matches) == 1:
            return click.Group.get_command(self, ctx, matches[0])
        ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
        return None


def debug_init():
    global debuglevel
    if os.path.exists(WARM_UPGRADE_DEBUG_FILE):
        debuglevel = debuglevel | WARMUPGRADEDEBUG
    else:
        debuglevel = debuglevel & ~(WARMUPGRADEDEBUG)


def warmupgradewarninglog(s):
    # s = s.decode('utf-8').encode('gb2312')
    syslog.openlog("WARMUPGRADE", syslog.LOG_PID)
    syslog.syslog(syslog.LOG_WARNING, s)


def warmupgradecriticallog(s):
    # s = s.decode('utf-8').encode('gb2312')
    syslog.openlog("WARMUPGRADE", syslog.LOG_PID)
    syslog.syslog(syslog.LOG_CRIT, s)


def warmupgradeerror(s):
    # s = s.decode('utf-8').encode('gb2312')
    syslog.openlog("WARMUPGRADE", syslog.LOG_PID)
    syslog.syslog(syslog.LOG_ERR, s)


def warmupgradedebuglog(s):
    # s = s.decode('utf-8').encode('gb2312')
    if WARMUPGRADEDEBUG & debuglevel:
        syslog.openlog("WARMUPGRADE", syslog.LOG_PID)
        syslog.syslog(syslog.LOG_DEBUG, s)


def subprocess_warm_upgrade(file, main_type, sub_type, slot):
    command = "firmware_upgrade %s 0x%x 0x%x %s" % (file, main_type, sub_type, slot)
    warmupgradedebuglog("warm upgrade firmware cmd:%s" % command)
    if os.path.exists(WARM_UPGRADE_DEBUG_FILE):
        return exec_os_cmd_log(command)
    return exec_os_cmd(command)


def signal_init():
    signal.signal(signal.SIGINT, signal.SIG_IGN)  # ignore ctrl+c signal
    signal.signal(signal.SIGTERM, signal.SIG_IGN)  # ignore kill signal
    signal.signal(signal.SIGTSTP, signal.SIG_IGN)  # ignore ctrl+z signal


class RefreshUpgradeBase(object):

    def __init__(self, config, slot_num, devtype, subtype):
        self._config = config
        self._slot_num = slot_num
        self._devtype = devtype
        self._subtype = subtype
        self.device_name = self._config.get("name", None)
        self.refresh_file = self._config.get("refresh_file", None)
        self.init_cmd_list = self._config.get("init_cmd", [])
        self.save_set_reg_list = self._config.get("save_set_reg", [])
        self.rw_recover_reg_list = self._config.get("rw_recover_reg", [])
        self.after_upgrade_delay = self._config.get("after_upgrade_delay", None)
        self.after_upgrade_delay_timeout = self._config.get("after_upgrade_delay_timeout", None)
        self.refresh_finish_flag_check_config = self._config.get("refresh_finish_flag_check", None)
        self.access_check_reg_config = self._config.get("access_check_reg", {})
        self.time_delay = 0
        self.finish_cmd_list = self._config.get("finish_cmd", [])

    def get_config(self):
        pass

    def get_slot_num(self):
        pass

    def save_value(self, cfg_list):
        for config in cfg_list:
            ret, val = get_value(config)
            if ret:
                config["value"] = val
            else:
                warmupgradeerror(val)
                return False, val
        return True, "save value success"

    def save_and_set_value(self, cfg_list):
        for config in cfg_list:
            ret, val = get_value(config)
            if ret:
                config["save_value"] = val
            else:
                warmupgradeerror(val)
                return False, "get save value fail"
            set_val = config.get("set_value", None)
            if set_val is not None:
                config["value"] = set_val
            else:
                warmupgradeerror("save_and_set_value lack of set_val config")
                return False, "set value is not config"
            ret, log = set_value(config)
            if ret is False:
                warmupgradeerror(log)
                return False, log
        return True, "save value success"

    def recover_value(self, cfg_list):
        fail_flag = 0
        for config in cfg_list:
            ret, log = set_value(config)
            if ret is False:
                fail_flag = -1
                warmupgradeerror("recover_value set_value failed, log: %s" % log)
        if fail_flag != 0:
            warmupgradeerror("recover_value write failed")
            return False, "recover write failed"
        return True, "recover write success"

    def recover_save_value(self, cfg_list):
        total_err = 0
        for config in cfg_list:
            val = config.get("save_value", None)
            if val is None:
                warmupgradeerror("recover_save_value lack of save_value config")
                total_err -= 1
                continue
            config["value"] = val
            ret, log = set_value(config)
            if ret is False:
                total_err -= 1
                warmupgradeerror("recover save value write failed, log: %s" % log)
            else:
                warmupgradedebuglog("recover save value success")
        if total_err < 0:
            return False, "recover save value failed"
        return True, "recover save value success"

    def do_fw_upg_init_cmd(self, init_cmd_list):
        # pre operation
        try:
            for init_cmd_config in init_cmd_list:
                ret, log = set_value(init_cmd_config)
                if ret is False:
                    warmupgradeerror("%s do init cmd: %s failed, msg: %s" % (self.device_name, init_cmd_config, log))
                    return False, log
            msg = "%s warm upgrade init cmd all set success" % self.device_name
            warmupgradedebuglog(msg)
            return True, msg
        except Exception as e:
            return False, str(e)

    def do_fw_upg_finish_cmd(self, finish_cmd_list):
        # end operation
        total_err = 0
        for finish_cmd_config in finish_cmd_list:
            ret_t, log = set_value(finish_cmd_config)
            if ret_t is False:
                warmupgradeerror("%s do finish cmd: %s failed, msg: %s" % (self.device_name, finish_cmd_config, log))
                total_err -= 1
        if total_err < 0:
            msg = "%s warm upgrade finish cmd exec failed" % self.device_name
            warmupgradeerror(msg)
            return False, msg
        msg = "%s warm upgrade finish cmd all set success" % self.device_name
        warmupgradedebuglog(msg)
        return True, msg

    def access_test(self, config):
        # polling execute command
        polling_cmd_list = config.get("polling_cmd", [])
        for polling_cmd_config in polling_cmd_list:
            ret, log = set_value(polling_cmd_config)
            if ret is False:
                warmupgradeerror(log)
                return False
        polling_delay = config.get("polling_delay", None)
        if polling_delay is not None:
            time.sleep(polling_delay)

        # record check val
        check_val = config.get("value", None)
        # write value
        ret, log = set_value(config)
        if ret is False:
            warmupgradeerror(log)
            return False
        # read value
        ret, val = get_value(config)
        if ret is False:
            warmupgradeerror(val)
            return False

        # compare write and read val
        warmupgradedebuglog("check_val:%s" % check_val)
        warmupgradedebuglog("get_value:%s" % val)
        if val != check_val:
            warmupgradeerror("check_val:%s != get_value:%s" % (check_val, val))
            return False
        return True

    def check_value(self, config):
        # record check val
        check_val = config.get("value", None)
        ret, val = get_value(config)
        if ret is False:
            warmupgradeerror(val)
            return False
        # compare write and read val
        warmupgradedebuglog("check_val:%s" % check_val)
        warmupgradedebuglog("get_value:%s" % val)
        if val != check_val:
            warmupgradeerror("check_val:%s != get_value:%s" % (check_val, val))
            return False
        return True

    def refresh_file_upgrade(self):
        try:
            warmupgradedebuglog("start %s warm upgrading" % self.device_name)

            # save and set reg
            ret, log = self.save_and_set_value(self.save_set_reg_list)
            if ret is False:
                warmupgradeerror(log)
                self.recover_save_value(self.save_set_reg_list)
                self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                return False, log
            warmupgradedebuglog("%s save and set reg cmd all set success" % self.device_name)
            time.sleep(0.5)  # delay 0.5s after execute save and set reg

            # pre operation
            ret, log = self.do_fw_upg_init_cmd(self.init_cmd_list)
            if ret is False:
                warmupgradeerror(log)
                self.recover_save_value(self.save_set_reg_list)
                self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                return False, log
            time.sleep(0.5)  # delay 0.5s after execute init_cmd

            # save reg
            ret, log = self.save_value(self.rw_recover_reg_list)
            if ret is False:
                warmupgradeerror("%s save reg failed" % self.device_name)
                self.recover_save_value(self.save_set_reg_list)
                self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                return False, log
            warmupgradedebuglog("%s all reg save success" % self.device_name)

            # upgrade refresh file
            if self.refresh_file is not None:
                status, output = subprocess_warm_upgrade(
                    self.refresh_file, self._devtype, self._subtype, self._slot_num)
                if status:
                    log = "%s refresh file upg failed, msg: %s" % (self.device_name, output)
                    warmupgradeerror(log)
                    self.recover_save_value(self.save_set_reg_list)
                    self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                    return False, log
                warmupgradedebuglog("%s refresh file upg success" % self.device_name)

            # delay the preset time after the upgrade is complete
            if self.after_upgrade_delay is not None:
                time.sleep(self.after_upgrade_delay)

            # check something in the timeout period
            if self.after_upgrade_delay_timeout is not None:
                while self.time_delay < self.after_upgrade_delay_timeout:

                    # check refresh finish flag
                    if self.refresh_finish_flag_check_config is not None:
                        ret = self.check_value(self.refresh_finish_flag_check_config)
                        if ret is False:
                            time.sleep(1)
                            self.time_delay = self.time_delay + 1
                            warmupgradedebuglog("doing refresh_finish_flag_check, time_delay:%s" % self.time_delay)
                            continue
                        warmupgradedebuglog("%s upgrade_finish_flag_check success. self.time_delay:%d"
                                            % (self.device_name, self.time_delay))

                    # doing logic device rw access test
                    ret = self.access_test(self.access_check_reg_config)
                    if ret:
                        warmupgradedebuglog(
                            "%s rw test success. self.time_delay:%d" %
                            (self.device_name, self.time_delay))
                        break
                    time.sleep(1)
                    self.time_delay = self.time_delay + 1
                    warmupgradedebuglog("doing access_test, self.time_delay:%s" % self.time_delay)

                if self.time_delay >= self.after_upgrade_delay_timeout:
                    log = "wait %s access test timeout" % self.device_name
                    warmupgradeerror(log)
                    self.recover_save_value(self.save_set_reg_list)
                    self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                    return False, log
                warmupgradedebuglog("%s access test success" % self.device_name)

            # recover reg
            ret, log = self.recover_value(self.rw_recover_reg_list)
            if ret is False:
                warmupgradeerror("recover %s reg failed" % self.device_name)
                self.recover_save_value(self.save_set_reg_list)
                self.do_fw_upg_finish_cmd(self.finish_cmd_list)
                return False, log
            warmupgradedebuglog("recover %s reg success" % self.device_name)
            # finally
            ret1, log1 = self.recover_save_value(self.save_set_reg_list)
            if ret1 is False:
                warmupgradeerror("bmc upgrade recover save value failed, msg: %s" % log1)
            ret2, log2 = self.do_fw_upg_finish_cmd(self.finish_cmd_list)
            if ret2 is False:
                warmupgradeerror("bmc upgrade do finish command failed, msg: %s" % log2)
            if ret1 is False or ret2 is False:
                return False, "upgrading %s recover save value or finish command failed" % self.device_name
            return True, "upgrading %s success" % self.device_name

        except Exception as e:
            log = "refresh file upgrade Exception happend, error log : %s" % str(e)
            self.recover_save_value(self.save_set_reg_list)
            self.do_fw_upg_finish_cmd(self.finish_cmd_list)
            return False, log


class RefreshUpgrade(RefreshUpgradeBase):

    def __init__(self, config, slot_num, devtype, subtype):
        super(RefreshUpgrade, self).__init__(config, slot_num, devtype, subtype)

    def get_config(self):
        super(RefreshUpgrade, self).get_config()
        return self._config

    def get_slot_num(self):
        super(RefreshUpgrade, self).get_slot_num()
        return self._slot_num


class WarmBasePlatform():

    def __init__(self):
        signal_init()
        debug_init()
        self.warm_upgrade_param = WARM_UPGRADE_PARAM.copy()
        self.stop_services_cmd_list = self.warm_upgrade_param.get("stop_services_cmd", [])
        self.start_services_cmd_list = self.warm_upgrade_param.get("start_services_cmd", [])
        self.__warm_upgrade_config_list = []

    def execute_command_list(self, cmd_list):
        for cmd_item in cmd_list:
            warmupgradedebuglog("execute cmd: %s" % cmd_item)
            status, output = exec_os_cmd(cmd_item)
            if status:
                log = "execute %s failed, msg: %s" % (cmd_item, output)
                warmupgradeerror(log)
                return False, log
        return True, "execute success"

    def stop_services_access(self):
        return self.execute_command_list(self.stop_services_cmd_list)

    def start_services_access(self):
        return self.execute_command_list(self.start_services_cmd_list)

    def check_slot_present(self, slot_present_config):
        totalerr = 0
        presentbit = slot_present_config.get('presentbit')
        ret, value = get_value(slot_present_config)
        if ret is False:
            return "NOT OK"
        if isinstance(value, str):
            val_t = int(value, 16)
        else:
            val_t = value
        val_t = (val_t & (1 << presentbit)) >> presentbit
        if val_t != slot_present_config.get('okval'):
            status = "ABSENT"
        else:
            status = "PRESENT"
        return status

    def linecard_present_check(self, slot_name, slot_present_config):
        present_status = self.check_slot_present(slot_present_config)
        present_status_tuple = ("ABSENT", "NOT OK")
        if present_status in present_status_tuple:
            return False, ("%s not present, warm upgrade exit" % slot_name)
        warmupgradedebuglog("%s present" % slot_name)
        return True, ("%s present" % slot_name)

    def start_warmupgrade(self):
        try:
            # start refresh file upgrade process
            for dev in self.__warm_upgrade_config_list:
                ret, log = dev.refresh_file_upgrade()
                if ret is False:
                    return ret, log
            return True, "all success"
        except Exception as e:
            log = "Exception happend, error log : %s" % str(e)
            return False, log

    def do_warmupgrade(self, file, main_type, sub_type, slot, file_type, chain):
        try:
            # upgrade file existence check
            if not os.path.isfile(file):
                return False, "%s not found" % file

            # get slot config
            slot_name = "slot%d" % slot
            slot_config = self.warm_upgrade_param.get(slot_name, {})
            if len(slot_config) == 0:
                return False, ("%s config not found" % slot_name)

            # linecard present check
            slot_present_config = slot_config.get("present", {})
            if len(slot_present_config) != 0:
                ret, log = self.linecard_present_check(slot_name, slot_present_config)
                if ret is False:
                    return False, log

            # match file_type and chain_num get chain_config
            file_type_config = slot_config.get(file_type, {})
            chain_name = "chain%d" % chain
            chain_list = file_type_config.get(chain_name, [])
            self.__warm_upgrade_config_list = []
            for refresh_config in chain_list:
                # refresh_file existence check
                refresh_file_judge_flag = refresh_config.get("refresh_file_judge_flag", 0)
                if refresh_file_judge_flag == 1:
                    refresh_file = refresh_config.get("refresh_file", None)
                    if not os.path.isfile(refresh_file):
                        log = "%s not found" % refresh_file
                        return False, log
                # each refresh_config add as an instance of RefreshUpgrade Class
                refresh_instance = RefreshUpgrade(refresh_config, slot, main_type, sub_type)
                self.__warm_upgrade_config_list.append(refresh_instance)

            ret, log = self.start_warmupgrade()
            if ret is False:
                warmupgradeerror("doing warm upgrade failed")
                warmupgradeerror(log)
                return ret, log

        except Exception as e:
            log = "Exception happend, error log : %s" % str(e)
            return False, log
        return True, "all success"

    def do_warm_upgrade(self, file, main_type, sub_type, slot, file_type, chain):
        print("+================================+")
        print("|Begin warm upgrade, please wait..|")
        ret, log = self.do_warmupgrade(file, main_type, sub_type, slot, file_type, chain)
        if ret:
            print("|     warm upgrade succeeded!    |")
            print("+================================+")
            sys.exit(0)
        else:
            print("|       warm upgrade failed!     |")
            print("+================================+")
            print("FAILED REASON:")
            print("%s" % log)
            sys.exit(1)


@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
@click.argument('file', required=True)
@click.argument('main_type', required=True)
@click.argument('sub_type', required=True)
@click.argument('slot', required=True)
@click.argument('file_type', required=True)
@click.argument('chain', required=True)
def main(file, main_type, sub_type, slot, file_type, chain):
    '''warm upgrade'''
    signal_init()
    debug_init()
    platform = WarmBasePlatform()
    platform.do_warm_upgrade(file, int(main_type, 16), int(sub_type, 16), int(slot), file_type, int(chain))


# warm upgrade
if __name__ == '__main__':
    main()