1013 lines
45 KiB
Python
1013 lines
45 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
import time
|
||
|
|
import syslog
|
||
|
|
import traceback
|
||
|
|
from plat_hal.interface import interface
|
||
|
|
from plat_hal.baseutil import baseutil
|
||
|
|
from algorithm.pid import pid
|
||
|
|
from algorithm.openloop import openloop
|
||
|
|
from algorithm.hysteresis import hysteresis
|
||
|
|
|
||
|
|
|
||
|
|
SWITCH_TEMP = "SWITCH_TEMP"
|
||
|
|
INLET_TEMP = "INLET_TEMP"
|
||
|
|
BOARD_TEMP = "BOARD_TEMP"
|
||
|
|
OUTLET_TEMP = "OUTLET_TEMP"
|
||
|
|
CPU_TEMP = "CPU_TEMP"
|
||
|
|
|
||
|
|
FANCTROL_DEBUG_FILE = "/etc/.fancontrol_debug_flag"
|
||
|
|
OTP_REBOOT_JUDGE_FILE = "/etc/.otp_reboot_flag" #coordination with REBOOT_CAUSE_PARA
|
||
|
|
|
||
|
|
FANCTROLERROR = 1
|
||
|
|
FANCTROLDEBUG = 2
|
||
|
|
FANAIRFLOWDEBUG = 4
|
||
|
|
|
||
|
|
debuglevel = 0
|
||
|
|
|
||
|
|
F2B_AIR_FLOW = "F2B"
|
||
|
|
B2F_AIR_FLOW = "B2F"
|
||
|
|
ONIE_E2_NAME = "ONIE_E2"
|
||
|
|
|
||
|
|
|
||
|
|
def fancontrol_debug(s):
|
||
|
|
if FANCTROLDEBUG & debuglevel:
|
||
|
|
syslog.openlog("FANCONTROL", syslog.LOG_PID)
|
||
|
|
syslog.syslog(syslog.LOG_DEBUG, s)
|
||
|
|
|
||
|
|
|
||
|
|
def fancontrol_error(s):
|
||
|
|
if FANCTROLERROR & debuglevel:
|
||
|
|
syslog.openlog("FANCONTROL", syslog.LOG_PID)
|
||
|
|
syslog.syslog(syslog.LOG_ERR, s)
|
||
|
|
|
||
|
|
|
||
|
|
def fanairflow_debug(s):
|
||
|
|
if FANAIRFLOWDEBUG & debuglevel:
|
||
|
|
syslog.openlog("AIR_FLOW_MONITOR", syslog.LOG_PID)
|
||
|
|
syslog.syslog(syslog.LOG_DEBUG, s)
|
||
|
|
|
||
|
|
|
||
|
|
def exec_os_cmd(cmd):
|
||
|
|
status, output = subprocess.getstatusoutput(cmd)
|
||
|
|
if status:
|
||
|
|
print(output)
|
||
|
|
return status, output
|
||
|
|
|
||
|
|
|
||
|
|
def debug_init():
|
||
|
|
global debuglevel
|
||
|
|
try:
|
||
|
|
with open(FANCTROL_DEBUG_FILE, "r") as fd:
|
||
|
|
value = fd.read()
|
||
|
|
debuglevel = int(value)
|
||
|
|
except Exception as e:
|
||
|
|
debuglevel = 0
|
||
|
|
|
||
|
|
|
||
|
|
error_temp = -9999 # get temp error
|
||
|
|
invalid_temp = -10000 # get temp invalid
|
||
|
|
PRE_FAN_NOK_UNKNOWN = "UNKNOWN"
|
||
|
|
|
||
|
|
|
||
|
|
class DevFan(object):
|
||
|
|
|
||
|
|
def __init__(self, name, interface):
|
||
|
|
self.__name = name
|
||
|
|
self.origin_name = None
|
||
|
|
self.display_name = None
|
||
|
|
self.air_flow = None
|
||
|
|
self.air_flow_inconsistent = False
|
||
|
|
self.int_case = interface
|
||
|
|
|
||
|
|
@property
|
||
|
|
def name(self):
|
||
|
|
return self.__name
|
||
|
|
|
||
|
|
def get_fan_rotor_number(self):
|
||
|
|
return self.int_case.get_fan_rotor_number(self.name)
|
||
|
|
|
||
|
|
def get_fan_presence(self):
|
||
|
|
return self.int_case.get_fan_presence(self.name)
|
||
|
|
|
||
|
|
def get_fan_rotor_status(self, rotor_name):
|
||
|
|
return self.int_case.get_fan_rotor_status(self.name, rotor_name)
|
||
|
|
|
||
|
|
def get_fan_fru_info(self):
|
||
|
|
return self.int_case.get_fan_fru_info(self.name)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def na_ret(self):
|
||
|
|
return self.int_case.na_ret
|
||
|
|
|
||
|
|
def update_fru_info(self):
|
||
|
|
try:
|
||
|
|
dic = self.get_fan_fru_info()
|
||
|
|
self.origin_name = dic["PN"]
|
||
|
|
self.air_flow = dic["AirFlow"]
|
||
|
|
self.display_name = dic["DisplayName"]
|
||
|
|
except Exception as e:
|
||
|
|
fanairflow_debug("update %s fru info error, msg: %s" % (self.name, str(e)))
|
||
|
|
self.origin_name = self.na_ret
|
||
|
|
self.air_flow = self.na_ret
|
||
|
|
self.display_name = self.na_ret
|
||
|
|
|
||
|
|
|
||
|
|
class fancontrol(object):
|
||
|
|
__int_case = None
|
||
|
|
|
||
|
|
__pwm = 0x80
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.int_case = interface()
|
||
|
|
self.__config = baseutil.get_monitor_config()
|
||
|
|
self.__pid_config = self.__config["pid"]
|
||
|
|
self.__hyst_config = self.__config.get("hyst", {})
|
||
|
|
self.__temps_threshold_config = self.__config["temps_threshold"]
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
temp_threshold['temp'] = 0
|
||
|
|
temp_threshold['fail_num'] = 0
|
||
|
|
temp_threshold['warning_num'] = 0 # temp warning times
|
||
|
|
temp_threshold['critical_num'] = 0 # temp critical times
|
||
|
|
temp_threshold['emergency_num'] = 0 # temp emergency times
|
||
|
|
temp_threshold.setdefault('ignore_threshold', 0) # default temp threshold on
|
||
|
|
temp_threshold.setdefault('invalid', invalid_temp)
|
||
|
|
temp_threshold.setdefault('error', error_temp)
|
||
|
|
|
||
|
|
self.__fan_rotor_error_num = {}
|
||
|
|
self.__fan_present_status = {} # {"FAN1":0, "FAN2":1...} 1:present, 0:absent
|
||
|
|
self.__fan_rotate_status = {} # {"FAN1":0, "FAN2":1...} 1:OK, 0:NOT OK
|
||
|
|
self.__fan_repair_flag = {} # {"FAN1":0, "FAN2":1...} 1:repair, 0:give up
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
self.__fan_present_status[fan_name] = 1 # present
|
||
|
|
self.__fan_rotate_status[fan_name] = 1 # OK
|
||
|
|
self.__fan_repair_flag[fan_name] = 1 # repair
|
||
|
|
rotor_num = self.get_rotor_number(fan_name)
|
||
|
|
tmp_fan = {}
|
||
|
|
for j in range(rotor_num):
|
||
|
|
rotor_name = "Rotor" + str(j + 1)
|
||
|
|
tmp_fan[rotor_name] = 0 # not error
|
||
|
|
self.__fan_rotor_error_num[fan_name] = tmp_fan
|
||
|
|
|
||
|
|
self.__fancontrol_para = self.__config["fancontrol_para"]
|
||
|
|
self.__interval = self.__fancontrol_para.get("interval", 5)
|
||
|
|
self.__fan_status_interval = self.__fancontrol_para.get("fan_status_interval", 0)
|
||
|
|
self.__max_pwm = self.__fancontrol_para.get("max_pwm", 0xff)
|
||
|
|
self.__min_pwm = self.__fancontrol_para.get("min_pwm", 0x80)
|
||
|
|
self.__abnormal_pwm = self.__fancontrol_para.get("abnormal_pwm", 0xbb)
|
||
|
|
self.__warning_pwm = self.__fancontrol_para.get("warning_pwm", 0xff)
|
||
|
|
self.__temp_invalid_pid_pwm = self.__fancontrol_para.get("temp_invalid_pid_pwm", 0x80)
|
||
|
|
self.__temp_error_pid_pwm = self.__fancontrol_para.get("temp_error_pid_pwm", 0x80)
|
||
|
|
self.__temp_fail_num = self.__fancontrol_para.get("temp_fail_num", 3)
|
||
|
|
self.__check_temp_fail = self.__fancontrol_para.get("check_temp_fail", [])
|
||
|
|
self.__temp_warning_num = self.__fancontrol_para.get("temp_warning_num", 3)
|
||
|
|
self.__temp_critical_num = self.__fancontrol_para.get("temp_critical_num", 3)
|
||
|
|
self.__temp_emergency_num = self.__fancontrol_para.get("temp_emergency_num", 3)
|
||
|
|
self.__temp_warning_countdown = self.__fancontrol_para.get("temp_warning_countdown", 60)
|
||
|
|
self.__temp_critical_countdown = self.__fancontrol_para.get("temp_critical_countdown", 60)
|
||
|
|
self.__temp_emergency_countdown = self.__fancontrol_para.get("temp_emergency_countdown", 60)
|
||
|
|
self.__rotor_error_count = self.__fancontrol_para.get("rotor_error_count", 6)
|
||
|
|
self.__inlet_mac_diff = self.__fancontrol_para.get("inlet_mac_diff", 50)
|
||
|
|
self.__check_crit_reboot_flag = self.__fancontrol_para.get("check_crit_reboot_flag", 1)
|
||
|
|
self.__check_emerg_reboot_flag = self.__fancontrol_para.get("check_emerg_reboot_flag", 1)
|
||
|
|
self.__check_crit_reboot_num = self.__fancontrol_para.get("check_crit_reboot_num", 3)
|
||
|
|
self.__check_crit_sleep_time = self.__fancontrol_para.get("check_crit_sleep_time", 20)
|
||
|
|
self.__check_emerg_reboot_num = self.__fancontrol_para.get("check_emerg_reboot_num", 3)
|
||
|
|
self.__check_emerg_sleep_time = self.__fancontrol_para.get("check_emerg_sleep_time", 20)
|
||
|
|
self.__check_temp_emergency = self.__fancontrol_para.get("check_temp_emergency", 0)
|
||
|
|
self.__check_temp_critical = self.__fancontrol_para.get("check_temp_critical", 1)
|
||
|
|
self.__check_temp_warning = self.__fancontrol_para.get("check_temp_warning", 1)
|
||
|
|
self.__check_temp_emergency_reboot = self.__fancontrol_para.get("check_temp_emergency_reboot", [])
|
||
|
|
self.__psu_absent_fullspeed_num = self.__fancontrol_para.get("psu_absent_fullspeed_num", 1)
|
||
|
|
self.__fan_absent_fullspeed_num = self.__fancontrol_para.get("fan_absent_fullspeed_num", 1)
|
||
|
|
self.__rotor_error_fullspeed_num = self.__fancontrol_para.get("rotor_error_fullspeed_num", 1)
|
||
|
|
self.__psu_fan_control = self.__fancontrol_para.get("psu_fan_control", 1) # default control psu fan
|
||
|
|
self.__fan_plug_in_pwm = self.__fancontrol_para.get("fan_plug_in_pwm", 0x80)
|
||
|
|
self.__fan_plug_in_default_countdown = self.__fancontrol_para.get("fan_plug_in_default_countdown", 0)
|
||
|
|
self.__deal_fan_error_policy = self.__fancontrol_para.get("deal_fan_error", 0)
|
||
|
|
self.__deal_fan_error_conf = self.__fancontrol_para.get("deal_fan_error_conf", {})
|
||
|
|
self.__deal_fan_error_default_countdown = self.__deal_fan_error_conf.get("countdown", 0)
|
||
|
|
|
||
|
|
self.__warning_countdown = 0 # temp warning flag for normal fancontrol
|
||
|
|
self.__critical_countdown = 0 # temp critical flag for normal fancontrol
|
||
|
|
self.__emergency_countdown = 0 # temp emergency flag for normal fancontrol
|
||
|
|
self.__fan_plug_in_countdown = 0 # fan plug in flag for normal fancontrol
|
||
|
|
self.__deal_fan_error_countdown = 0
|
||
|
|
self.__fan_absent_num = 0
|
||
|
|
self.__fan_nok_num = 0
|
||
|
|
self.__pre_fan_nok = PRE_FAN_NOK_UNKNOWN
|
||
|
|
self.openloop = openloop()
|
||
|
|
self.pid = pid()
|
||
|
|
self.hyst = hysteresis()
|
||
|
|
self.__pwm = self.__min_pwm
|
||
|
|
|
||
|
|
self.__board_air_flow = ""
|
||
|
|
self.__fan_air_flow_monitor = self.__fancontrol_para.get("fan_air_flow_monitor", 0)
|
||
|
|
self.__air_flow_correct_fan_pwm = self.__fancontrol_para.get("air_flow_correct_fan_pwm", 0xff)
|
||
|
|
self.__air_flow_error_fan_pwm = self.__fancontrol_para.get("air_flow_error_fan_pwm", 0)
|
||
|
|
self.__air_flow_error_psu_pwm = self.__fancontrol_para.get("air_flow_error_psu_pwm", 0xff)
|
||
|
|
self.air_flow_inconsistent_flag = False
|
||
|
|
self.fan_obj_list = []
|
||
|
|
|
||
|
|
@property
|
||
|
|
def na_ret(self):
|
||
|
|
return self.int_case.na_ret
|
||
|
|
|
||
|
|
def get_onie_e2_obj(self, name):
|
||
|
|
return self.int_case.get_onie_e2_obj(name)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def board_air_flow(self):
|
||
|
|
if self.__board_air_flow != F2B_AIR_FLOW and self.__board_air_flow != B2F_AIR_FLOW:
|
||
|
|
onie_e2_obj = self.get_onie_e2_obj(ONIE_E2_NAME)
|
||
|
|
if onie_e2_obj is not None:
|
||
|
|
fanairflow_debug("onie_e2 servicetag: %s" % onie_e2_obj.servicetag)
|
||
|
|
if isinstance(onie_e2_obj.servicetag, str) and onie_e2_obj.servicetag.startswith("F2B"):
|
||
|
|
self.__board_air_flow = F2B_AIR_FLOW
|
||
|
|
elif isinstance(onie_e2_obj.servicetag, str) and onie_e2_obj.servicetag.startswith("B2F"):
|
||
|
|
self.__board_air_flow = B2F_AIR_FLOW
|
||
|
|
else:
|
||
|
|
self.__board_air_flow = onie_e2_obj.servicetag
|
||
|
|
fanairflow_debug("board_air_flow: %s" % self.__board_air_flow)
|
||
|
|
return self.__board_air_flow
|
||
|
|
|
||
|
|
@property
|
||
|
|
def fan_air_flow_monitor(self):
|
||
|
|
return self.__fan_air_flow_monitor
|
||
|
|
|
||
|
|
@property
|
||
|
|
def air_flow_correct_fan_pwm(self):
|
||
|
|
return self.__air_flow_correct_fan_pwm
|
||
|
|
|
||
|
|
@property
|
||
|
|
def air_flow_error_fan_pwm(self):
|
||
|
|
return self.__air_flow_error_fan_pwm
|
||
|
|
|
||
|
|
@property
|
||
|
|
def air_flow_error_psu_pwm(self):
|
||
|
|
return self.__air_flow_error_psu_pwm
|
||
|
|
|
||
|
|
def get_para(self, type):
|
||
|
|
para = self.__pid_config.get(type)
|
||
|
|
return para
|
||
|
|
|
||
|
|
def update_over_temp_threshold_num(self):
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
if temp_threshold['ignore_threshold']:
|
||
|
|
continue
|
||
|
|
emergency_threshold = temp_threshold.get('emergency', None)
|
||
|
|
critical_threshold = temp_threshold.get('critical', None)
|
||
|
|
warning_threshold = temp_threshold.get('warning', None)
|
||
|
|
fancontrol_debug("%s warning = %s, critical = %s, emergency = %s" %
|
||
|
|
(temp_threshold['name'], warning_threshold, critical_threshold, emergency_threshold))
|
||
|
|
|
||
|
|
if emergency_threshold is not None and temp_threshold['temp'] >= emergency_threshold:
|
||
|
|
temp_threshold['emergency_num'] += 1
|
||
|
|
else:
|
||
|
|
temp_threshold['emergency_num'] = 0
|
||
|
|
|
||
|
|
if critical_threshold is not None and temp_threshold['temp'] >= critical_threshold:
|
||
|
|
temp_threshold['critical_num'] += 1
|
||
|
|
else:
|
||
|
|
temp_threshold['critical_num'] = 0
|
||
|
|
|
||
|
|
if warning_threshold is not None and temp_threshold['temp'] >= warning_threshold:
|
||
|
|
temp_threshold['warning_num'] += 1
|
||
|
|
else:
|
||
|
|
temp_threshold['warning_num'] = 0
|
||
|
|
|
||
|
|
fancontrol_debug("%s warning_num = %d, critical_num = %d, emergency_num = %d" %
|
||
|
|
(temp_threshold['name'], temp_threshold['warning_num'], temp_threshold['critical_num'], temp_threshold.get("emergency_num")))
|
||
|
|
return
|
||
|
|
|
||
|
|
def get_monitor_temp(self):
|
||
|
|
sensorlist = self.int_case.get_temp_info()
|
||
|
|
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
sensor = sensorlist.get(temp_threshold['name'])
|
||
|
|
if sensor["Value"] is None or int(sensor["Value"]) == self.int_case.error_ret:
|
||
|
|
temp_threshold['fail_num'] += 1
|
||
|
|
fancontrol_error("get %s failed, fail_num = %d" % (temp_threshold['name'], temp_threshold['fail_num']))
|
||
|
|
else:
|
||
|
|
temp_threshold['fail_num'] = 0
|
||
|
|
temp_threshold.setdefault('fix', 0)
|
||
|
|
temp_threshold['temp'] = sensor["Value"] + temp_threshold['fix']
|
||
|
|
fancontrol_debug("%s = %d" % (temp_threshold['name'], temp_threshold['temp']))
|
||
|
|
self.update_over_temp_threshold_num()
|
||
|
|
|
||
|
|
def is_temp_warning(self):
|
||
|
|
warning_flag = False
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
if temp_threshold['ignore_threshold']:
|
||
|
|
continue
|
||
|
|
if temp_threshold['warning_num'] >= self.__temp_warning_num:
|
||
|
|
warning_flag = True
|
||
|
|
fancontrol_debug("%s is over warning" % temp_threshold['name'])
|
||
|
|
fancontrol_debug("%s = %d, warning = %s" %
|
||
|
|
(temp_threshold['name'], temp_threshold['temp'], temp_threshold.get('warning')))
|
||
|
|
return warning_flag
|
||
|
|
|
||
|
|
def checkTempWarning(self):
|
||
|
|
try:
|
||
|
|
if self.is_temp_warning():
|
||
|
|
self.__warning_countdown = self.__temp_warning_countdown
|
||
|
|
fancontrol_debug("temp is over warning")
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
if self.__warning_countdown > 0:
|
||
|
|
self.__warning_countdown -= 1
|
||
|
|
return False
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkTempWarning failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkTempWarningCountdown(self):
|
||
|
|
if self.__warning_countdown > 0:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def is_temp_critical(self):
|
||
|
|
critical_flag = False
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
temp_threshold['critical_flag'] = False
|
||
|
|
if temp_threshold['ignore_threshold']:
|
||
|
|
continue
|
||
|
|
if temp_threshold['critical_num'] >= self.__temp_critical_num:
|
||
|
|
critical_flag = True
|
||
|
|
temp_threshold['critical_flag'] = True
|
||
|
|
fancontrol_debug("%s is over critical" % temp_threshold['name'])
|
||
|
|
fancontrol_debug("%s = %d, critical = %s" %
|
||
|
|
(temp_threshold['name'], temp_threshold['temp'], temp_threshold.get('critical')))
|
||
|
|
return critical_flag
|
||
|
|
|
||
|
|
def checkTempCritical(self):
|
||
|
|
try:
|
||
|
|
if self.is_temp_critical():
|
||
|
|
self.__critical_countdown = self.__temp_critical_countdown
|
||
|
|
fancontrol_debug("temp is over critical")
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
if self.__critical_countdown > 0:
|
||
|
|
self.__critical_countdown -= 1
|
||
|
|
return False
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkTempCrit failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
return False
|
||
|
|
|
||
|
|
def is_temp_emergency(self):
|
||
|
|
emergency_flag = False
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
temp_threshold['emergency_flag'] = False
|
||
|
|
if temp_threshold['ignore_threshold']:
|
||
|
|
continue
|
||
|
|
if temp_threshold['emergency_num'] >= self.__temp_emergency_num:
|
||
|
|
emergency_flag = True
|
||
|
|
temp_threshold['emergency_flag'] = True
|
||
|
|
fancontrol_debug("%s is over emergency" % temp_threshold['name'])
|
||
|
|
fancontrol_debug("%s = %d, emergency = %s" %
|
||
|
|
(temp_threshold['name'], temp_threshold['temp'], temp_threshold.get('emergency')))
|
||
|
|
return emergency_flag
|
||
|
|
|
||
|
|
|
||
|
|
def checkTempEmergency(self):
|
||
|
|
try:
|
||
|
|
if self.is_temp_emergency():
|
||
|
|
self.__emergency_countdown = self.__temp_emergency_countdown
|
||
|
|
fancontrol_debug("temp is over emergency")
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
if self.__emergency_countdown > 0:
|
||
|
|
self.__emergency_countdown -= 1
|
||
|
|
return False
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkTempEmergency failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkTempCriticalCountdown(self):
|
||
|
|
if self.__critical_countdown > 0:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkTempEmergencyCountdown(self):
|
||
|
|
if self.__emergency_countdown > 0:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkTempRebootCrit(self):
|
||
|
|
try:
|
||
|
|
if self.is_temp_critical():
|
||
|
|
temp_dict = dict(self.__temps_threshold_config)
|
||
|
|
tmp = temp_dict.get(SWITCH_TEMP)
|
||
|
|
if tmp['critical_flag'] == True:
|
||
|
|
fancontrol_debug("switch temp is over reboot critical")
|
||
|
|
return True
|
||
|
|
del temp_dict[SWITCH_TEMP]
|
||
|
|
for temp_items in temp_dict.values():
|
||
|
|
if temp_items['ignore_threshold']:
|
||
|
|
continue
|
||
|
|
if temp_items['critical_flag'] == False:
|
||
|
|
return False
|
||
|
|
|
||
|
|
fancontrol_debug("other temp is over reboot critical")
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkTempRebootCrit failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkCritReboot(self):
|
||
|
|
try:
|
||
|
|
reboot_flag = False
|
||
|
|
if self.checkTempRebootCrit() == True:
|
||
|
|
self.set_all_fan_speed_pwm(self.__max_pwm)
|
||
|
|
for i in range(self.__check_crit_reboot_num):
|
||
|
|
time.sleep(self.__check_crit_sleep_time)
|
||
|
|
self.get_monitor_temp()
|
||
|
|
if self.checkTempRebootCrit() == True:
|
||
|
|
fancontrol_debug("The temperature of device is over reboot critical value.")
|
||
|
|
reboot_flag = True
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
fancontrol_debug("The temperature of device is not over reboot critical value.")
|
||
|
|
reboot_flag = False
|
||
|
|
break
|
||
|
|
if reboot_flag is True:
|
||
|
|
reboot_log = "The temperature of device is over critical value."
|
||
|
|
reboot_log_cmd = "echo '%s' > /dev/ttyS0" % reboot_log
|
||
|
|
fancontrol_error(reboot_log)
|
||
|
|
exec_os_cmd(reboot_log_cmd)
|
||
|
|
reboot_log = "The system is going to reboot now."
|
||
|
|
reboot_log_cmd = "echo '%s' > /dev/ttyS0" % reboot_log
|
||
|
|
fancontrol_error(reboot_log)
|
||
|
|
exec_os_cmd(reboot_log_cmd)
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
fancontrol_error("%s = %d" % (temp_threshold['name'], temp_threshold['temp']))
|
||
|
|
create_judge_file = "touch %s" % OTP_REBOOT_JUDGE_FILE
|
||
|
|
exec_os_cmd(create_judge_file)
|
||
|
|
exec_os_cmd("sync")
|
||
|
|
time.sleep(3)
|
||
|
|
os.system("/sbin/reboot")
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkCritReboot failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
|
||
|
|
def checkTempRebootEmerg(self):
|
||
|
|
try:
|
||
|
|
if self.is_temp_emergency():
|
||
|
|
temp_emerg_reboot_flag = False
|
||
|
|
for temp_list in self.__check_temp_emergency_reboot:
|
||
|
|
for temp in temp_list:
|
||
|
|
tmp = self.__temps_threshold_config.get(temp)
|
||
|
|
if tmp['emergency_flag'] is False:
|
||
|
|
fancontrol_debug("temp_list %s, temp: %s not emergency" % (temp_list, temp))
|
||
|
|
temp_emerg_reboot_flag = False
|
||
|
|
break
|
||
|
|
temp_emerg_reboot_flag = True
|
||
|
|
if temp_emerg_reboot_flag is True:
|
||
|
|
fancontrol_debug("temp_list %s, all temp is over emergency reboot" % temp_list)
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkTempRebootEmerg failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
return False
|
||
|
|
|
||
|
|
def checkEmergReboot(self):
|
||
|
|
try:
|
||
|
|
reboot_flag = False
|
||
|
|
if self.checkTempRebootEmerg() is True:
|
||
|
|
self.set_all_fan_speed_pwm(self.__max_pwm)
|
||
|
|
for i in range(self.__check_emerg_reboot_num):
|
||
|
|
time.sleep(self.__check_emerg_sleep_time)
|
||
|
|
self.get_monitor_temp()
|
||
|
|
if self.checkTempRebootEmerg() is True:
|
||
|
|
fancontrol_debug("The temperature of device is over reboot emergency value, i = %d" % (i+1))
|
||
|
|
reboot_flag = True
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
fancontrol_debug("The temperature of device is not over reboot emergency value.")
|
||
|
|
reboot_flag = False
|
||
|
|
break
|
||
|
|
if reboot_flag is True:
|
||
|
|
reboot_log = "The temperature of device is over emergency value."
|
||
|
|
reboot_log_cmd = "echo '%s' > /dev/ttyS0" % reboot_log
|
||
|
|
fancontrol_error(reboot_log)
|
||
|
|
exec_os_cmd(reboot_log_cmd)
|
||
|
|
reboot_log = "The system is going to reboot now."
|
||
|
|
reboot_log_cmd = "echo '%s' > /dev/ttyS0" % reboot_log
|
||
|
|
fancontrol_error(reboot_log)
|
||
|
|
exec_os_cmd(reboot_log_cmd)
|
||
|
|
for temp_threshold in self.__temps_threshold_config.values():
|
||
|
|
fancontrol_error("%s = %d" % (temp_threshold['name'], temp_threshold['temp']))
|
||
|
|
create_judge_file = "touch %s" % OTP_REBOOT_JUDGE_FILE
|
||
|
|
exec_os_cmd(create_judge_file)
|
||
|
|
exec_os_cmd("sync")
|
||
|
|
time.sleep(3)
|
||
|
|
os.system("/sbin/reboot")
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: checkEmergReboot failed")
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
|
||
|
|
def get_fan_total_number(self):
|
||
|
|
return self.int_case.get_fan_total_number()
|
||
|
|
|
||
|
|
def get_rotor_number(self, fan_name):
|
||
|
|
return self.int_case.get_fan_rotor_number(fan_name)
|
||
|
|
|
||
|
|
def get_fan_presence(self, fan_name):
|
||
|
|
return self.int_case.get_fan_presence(fan_name)
|
||
|
|
|
||
|
|
def get_fan_rotor_status(self, fan_name, rotor_name):
|
||
|
|
return self.int_case.get_fan_rotor_status(fan_name, rotor_name)
|
||
|
|
|
||
|
|
def get_psu_total_number(self):
|
||
|
|
return self.int_case.get_psu_total_number()
|
||
|
|
|
||
|
|
def get_psu_presence(self, psu_name):
|
||
|
|
return self.int_case.get_psu_presence(psu_name)
|
||
|
|
|
||
|
|
def get_psu_input_output_status(self, psu_name):
|
||
|
|
return self.int_case.get_psu_input_output_status(psu_name)
|
||
|
|
|
||
|
|
def checkFanPresence(self):
|
||
|
|
absent_num = 0
|
||
|
|
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
rotor_num = self.get_rotor_number(fan_name)
|
||
|
|
tmp_fan = self.__fan_rotor_error_num.get(fan_name)
|
||
|
|
status = self.get_fan_presence(fan_name)
|
||
|
|
if status == False:
|
||
|
|
absent_num = absent_num + 1
|
||
|
|
self.__fan_present_status[fan_name] = 0
|
||
|
|
fancontrol_debug("%s absent" % fan_name)
|
||
|
|
else:
|
||
|
|
if self.__fan_present_status[fan_name] == 0: # absent -> present
|
||
|
|
self.__pre_fan_nok = PRE_FAN_NOK_UNKNOWN
|
||
|
|
self.__fan_plug_in_countdown = self.__fan_plug_in_default_countdown
|
||
|
|
self.__fan_repair_flag[fan_name] = 1
|
||
|
|
for j in range(rotor_num):
|
||
|
|
rotor_name = "Rotor" + str(j + 1)
|
||
|
|
tmp_fan[rotor_name] = 0
|
||
|
|
self.__fan_present_status[fan_name] = 1
|
||
|
|
fancontrol_debug("%s presence" % fan_name)
|
||
|
|
return absent_num
|
||
|
|
|
||
|
|
def checkFanRotorStatus(self):
|
||
|
|
err_num = 0
|
||
|
|
self.__fan_nok_num = 0
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
rotor_num = self.get_rotor_number(fan_name)
|
||
|
|
tmp_fan = self.__fan_rotor_error_num.get(fan_name)
|
||
|
|
fan_rotor_err_cnt = 0
|
||
|
|
for j in range(rotor_num):
|
||
|
|
rotor_name = "Rotor" + str(j + 1)
|
||
|
|
status = self.get_fan_rotor_status(fan_name, rotor_name)
|
||
|
|
if status == True:
|
||
|
|
tmp_fan[rotor_name] = 0
|
||
|
|
fancontrol_debug("%s %s ok" % (fan_name, rotor_name))
|
||
|
|
else:
|
||
|
|
tmp_fan[rotor_name] += 1
|
||
|
|
if tmp_fan[rotor_name] >= self.__rotor_error_count:
|
||
|
|
err_num = err_num + 1
|
||
|
|
fan_rotor_err_cnt += 1
|
||
|
|
fancontrol_debug("%s %s error" % (fan_name, rotor_name))
|
||
|
|
fancontrol_debug("%s %s error %d times" % (fan_name, rotor_name, tmp_fan[rotor_name]))
|
||
|
|
if fan_rotor_err_cnt == 0:
|
||
|
|
self.__fan_rotate_status[fan_name] = 1 # FAN is ok
|
||
|
|
else:
|
||
|
|
self.__fan_rotate_status[fan_name] = 0 # FAN is not ok
|
||
|
|
self.__fan_nok_num += 1
|
||
|
|
fancontrol_debug("fan not ok number:%d." % self.__fan_nok_num)
|
||
|
|
return err_num
|
||
|
|
|
||
|
|
def checkPsuPresence(self):
|
||
|
|
absent_num = 0
|
||
|
|
psu_num = self.get_psu_total_number()
|
||
|
|
for i in range(psu_num):
|
||
|
|
psu_name = "PSU" + str(i + 1)
|
||
|
|
status = self.get_psu_presence(psu_name)
|
||
|
|
if status == False:
|
||
|
|
absent_num = absent_num + 1
|
||
|
|
fancontrol_debug("%s absent" % psu_name)
|
||
|
|
else:
|
||
|
|
fancontrol_debug("%s presence" % psu_name)
|
||
|
|
return absent_num
|
||
|
|
|
||
|
|
def checkPsuStatus(self):
|
||
|
|
err_num = 0
|
||
|
|
psu_num = self.get_psu_total_number()
|
||
|
|
for i in range(psu_num):
|
||
|
|
psu_name = "PSU" + str(i + 1)
|
||
|
|
status = self.get_psu_input_output_status(psu_name)
|
||
|
|
if status == False:
|
||
|
|
err_num = err_num + 1
|
||
|
|
fancontrol_debug("%s error" % psu_name)
|
||
|
|
else:
|
||
|
|
fancontrol_debug("%s ok" % psu_name)
|
||
|
|
return err_num
|
||
|
|
|
||
|
|
def checkDevError(self):
|
||
|
|
pwm = self.__min_pwm
|
||
|
|
switchtemp = self.__temps_threshold_config.get(SWITCH_TEMP)['temp']
|
||
|
|
inlettemp = self.__temps_threshold_config.get(INLET_TEMP)['temp']
|
||
|
|
temp_diff = abs(switchtemp - inlettemp)
|
||
|
|
fancontrol_debug("|switchtemp - inlettemp| = %d" % temp_diff)
|
||
|
|
if temp_diff >= self.__inlet_mac_diff:
|
||
|
|
fancontrol_debug("temp_diff is over than inlet_mac_diff(%d)" % self.__inlet_mac_diff)
|
||
|
|
if self.__pwm > self.__abnormal_pwm:
|
||
|
|
pwm = self.__max_pwm
|
||
|
|
else:
|
||
|
|
pwm = self.__abnormal_pwm
|
||
|
|
return pwm
|
||
|
|
|
||
|
|
def checktempfail(self):
|
||
|
|
pwm = self.__min_pwm
|
||
|
|
for temp in self.__check_temp_fail:
|
||
|
|
temp_name = temp.get("temp_name")
|
||
|
|
temp_fail_num = self.__temps_threshold_config.get(temp_name)['fail_num']
|
||
|
|
if temp_fail_num >= self.__temp_fail_num:
|
||
|
|
pwm = self.__abnormal_pwm
|
||
|
|
fancontrol_debug("%s temp_fail_num = %d" % (temp_name, temp_fail_num))
|
||
|
|
fancontrol_debug("self.__temp_fail_num = %d" % self.__temp_fail_num)
|
||
|
|
return pwm
|
||
|
|
|
||
|
|
def abnormal_check(self):
|
||
|
|
pwm_list = []
|
||
|
|
pwm_min = self.__min_pwm
|
||
|
|
pwm_list.append(pwm_min)
|
||
|
|
|
||
|
|
if self.__check_temp_emergency == 1:
|
||
|
|
status = self.checkTempEmergency()
|
||
|
|
if status is True:
|
||
|
|
over_emerg_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(over_emerg_pwm)
|
||
|
|
fancontrol_debug("over_emerg_pwm = 0x%x" % over_emerg_pwm)
|
||
|
|
# do reset check
|
||
|
|
if self.__check_emerg_reboot_flag == 1:
|
||
|
|
self.checkEmergReboot()
|
||
|
|
else:
|
||
|
|
if self.checkTempEmergencyCountdown() == True: # temp lower than emergency in 5 min
|
||
|
|
over_emerg_countdown_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(over_emerg_countdown_pwm)
|
||
|
|
fancontrol_debug("TempEmergencyCountdown: %d, over_emerg_countdown_pwm = 0x%x" %
|
||
|
|
(self.__emergency_countdown, over_emerg_countdown_pwm))
|
||
|
|
|
||
|
|
if self.__check_temp_critical == 1:
|
||
|
|
status = self.checkTempCritical()
|
||
|
|
if status == True:
|
||
|
|
over_crit_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(over_crit_pwm)
|
||
|
|
fancontrol_debug("over_crit_pwm = 0x%x" % over_crit_pwm)
|
||
|
|
# do reset check
|
||
|
|
if self.__check_crit_reboot_flag == 1:
|
||
|
|
self.checkCritReboot()
|
||
|
|
else:
|
||
|
|
if self.checkTempCriticalCountdown() == True: # temp lower than critical in 5 min
|
||
|
|
over_crit_countdown_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(over_crit_countdown_pwm)
|
||
|
|
fancontrol_debug("TempCriticalCountdown: %d, over_crit_countdown_pwm = 0x%x" %
|
||
|
|
(self.__critical_countdown, over_crit_countdown_pwm))
|
||
|
|
|
||
|
|
if self.__check_temp_warning == 1:
|
||
|
|
status = self.checkTempWarning()
|
||
|
|
if status == True:
|
||
|
|
over_warn_pwm = self.__warning_pwm
|
||
|
|
pwm_list.append(over_warn_pwm)
|
||
|
|
fancontrol_debug("over_warn_pwm = 0x%x" % over_warn_pwm)
|
||
|
|
else:
|
||
|
|
if self.checkTempWarningCountdown() == True: # temp lower than warning in 5 min
|
||
|
|
over_warn_countdown_pwm = self.__warning_pwm
|
||
|
|
pwm_list.append(over_warn_countdown_pwm)
|
||
|
|
fancontrol_debug("TempWarningCountdown: %d, over_warn_countdown_pwm = 0x%x" %
|
||
|
|
(self.__warning_countdown, over_warn_countdown_pwm))
|
||
|
|
|
||
|
|
self.__fan_absent_num = self.checkFanPresence()
|
||
|
|
if self.__fan_absent_num >= self.__fan_absent_fullspeed_num:
|
||
|
|
fan_absent_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(fan_absent_pwm)
|
||
|
|
fancontrol_debug("fan_absent_pwm = 0x%x" % fan_absent_pwm)
|
||
|
|
|
||
|
|
rotor_err_num = self.checkFanRotorStatus()
|
||
|
|
if rotor_err_num >= self.__rotor_error_fullspeed_num:
|
||
|
|
rotor_err_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(rotor_err_pwm)
|
||
|
|
fancontrol_debug("rotor_err_pwm = 0x%x" % rotor_err_pwm)
|
||
|
|
|
||
|
|
psu_absent_num = self.checkPsuPresence()
|
||
|
|
if psu_absent_num >= self.__psu_absent_fullspeed_num:
|
||
|
|
psu_absent_pwm = self.__max_pwm
|
||
|
|
pwm_list.append(psu_absent_pwm)
|
||
|
|
fancontrol_debug("psu_absent_pwm = 0x%x" % psu_absent_pwm)
|
||
|
|
|
||
|
|
dev_err_pwm = self.checkDevError()
|
||
|
|
pwm_list.append(dev_err_pwm)
|
||
|
|
fancontrol_debug("dev_err_pwm = 0x%x" % dev_err_pwm)
|
||
|
|
|
||
|
|
temp_fail_pwm = self.checktempfail()
|
||
|
|
pwm_list.append(temp_fail_pwm)
|
||
|
|
fancontrol_debug("temp_fail_pwm = 0x%x" % temp_fail_pwm)
|
||
|
|
|
||
|
|
pwm = max(pwm_list)
|
||
|
|
return pwm
|
||
|
|
|
||
|
|
def get_error_fan(self):
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
if self.__fan_rotate_status[fan_name] == 0:
|
||
|
|
return fan_name
|
||
|
|
return None
|
||
|
|
|
||
|
|
def fan_error_update_pwm(self, fan_pwm_dict):
|
||
|
|
try:
|
||
|
|
fancontrol_debug("enter deal fan error policy")
|
||
|
|
ori_fan_pwm_dict = fan_pwm_dict.copy()
|
||
|
|
|
||
|
|
err_fan_name = self.get_error_fan()
|
||
|
|
if err_fan_name is None:
|
||
|
|
fancontrol_debug("fan name is None, do nothing.")
|
||
|
|
return ori_fan_pwm_dict
|
||
|
|
|
||
|
|
if self.__fan_repair_flag[err_fan_name] == 0:
|
||
|
|
fancontrol_debug("%s already repaired, do nothing." % err_fan_name)
|
||
|
|
return ori_fan_pwm_dict
|
||
|
|
|
||
|
|
if self.__pre_fan_nok != err_fan_name:
|
||
|
|
fancontrol_debug(
|
||
|
|
"not ok fan change from %s to %s, update countdown." %
|
||
|
|
(self.__pre_fan_nok, err_fan_name))
|
||
|
|
self.__deal_fan_error_countdown = self.__deal_fan_error_default_countdown
|
||
|
|
if self.__pre_fan_nok != PRE_FAN_NOK_UNKNOWN:
|
||
|
|
fancontrol_debug(
|
||
|
|
"%s repaire success, %s NOT OK, try to repaire." %
|
||
|
|
(self.__pre_fan_nok, err_fan_name))
|
||
|
|
self.__fan_repair_flag[self.__pre_fan_nok] = 0
|
||
|
|
self.__pre_fan_nok = err_fan_name
|
||
|
|
|
||
|
|
if self.__deal_fan_error_countdown > 0:
|
||
|
|
self.__deal_fan_error_countdown -= 1
|
||
|
|
fancontrol_debug("%s repaire, countdown %d." % (err_fan_name, self.__deal_fan_error_countdown))
|
||
|
|
|
||
|
|
if self.__deal_fan_error_countdown == 0:
|
||
|
|
self.__fan_repair_flag[err_fan_name] = 0
|
||
|
|
fancontrol_debug("%s set repaire fail flag, use origin pwm." % err_fan_name)
|
||
|
|
return ori_fan_pwm_dict
|
||
|
|
|
||
|
|
fan_err_pwm_conf_list = self.__deal_fan_error_conf[err_fan_name]
|
||
|
|
for item in fan_err_pwm_conf_list:
|
||
|
|
fan_pwm_dict[item["name"]] = item["pwm"]
|
||
|
|
fancontrol_debug("fan pwm update, fan pwm dict:%s" % fan_pwm_dict)
|
||
|
|
|
||
|
|
return fan_pwm_dict
|
||
|
|
except Exception as e:
|
||
|
|
fancontrol_error("%%policy: deal_fan_error raise Exception:%s" % str(e))
|
||
|
|
self.__pre_fan_nok = PRE_FAN_NOK_UNKNOWN
|
||
|
|
return ori_fan_pwm_dict
|
||
|
|
|
||
|
|
def get_fan_pwm_dict(self, default_pwm):
|
||
|
|
fan_pwm_dict = {}
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
fan_pwm_dict[fan_name] = default_pwm
|
||
|
|
if self.__deal_fan_error_policy:
|
||
|
|
if self.__fan_absent_num == 0 and self.__fan_nok_num == 1:
|
||
|
|
fan_pwm_dict = self.fan_error_update_pwm(fan_pwm_dict)
|
||
|
|
else:
|
||
|
|
if self.__pre_fan_nok != PRE_FAN_NOK_UNKNOWN and self.__fan_rotate_status[self.__pre_fan_nok] == 1:
|
||
|
|
fancontrol_debug("%s repaire success." % (self.__pre_fan_nok))
|
||
|
|
self.__fan_repair_flag[self.__pre_fan_nok] = 0
|
||
|
|
self.__pre_fan_nok = PRE_FAN_NOK_UNKNOWN
|
||
|
|
return fan_pwm_dict
|
||
|
|
|
||
|
|
def check_board_air_flow(self):
|
||
|
|
board_air_flow = self.board_air_flow
|
||
|
|
if board_air_flow != F2B_AIR_FLOW and board_air_flow != B2F_AIR_FLOW:
|
||
|
|
fanairflow_debug("get board air flow error, value [%s]" % board_air_flow)
|
||
|
|
return False
|
||
|
|
fanairflow_debug("board air flow check ok: %s" % board_air_flow)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def check_fan_air_flow(self):
|
||
|
|
if self.fan_air_flow_monitor:
|
||
|
|
fanairflow_debug("open air flow monitor, check fan air flow")
|
||
|
|
ret = self.check_board_air_flow()
|
||
|
|
if ret is False:
|
||
|
|
fanairflow_debug("get board air flow error, set air_flow_inconsistent_flag False")
|
||
|
|
self.air_flow_inconsistent_flag = False
|
||
|
|
return
|
||
|
|
air_flow_inconsistent_flag_tmp = False
|
||
|
|
for fan_obj in self.fan_obj_list:
|
||
|
|
fan_obj.update_fru_info()
|
||
|
|
fanairflow_debug("%s origin name: [%s], display name: [%s] air flow [%s]" %
|
||
|
|
(fan_obj.name, fan_obj.origin_name, fan_obj.display_name, fan_obj.air_flow))
|
||
|
|
if fan_obj.air_flow == self.na_ret:
|
||
|
|
fanairflow_debug("%s get air flow failed, set air_flow_inconsistent flag False" % fan_obj.name)
|
||
|
|
fan_obj.air_flow_inconsistent = False
|
||
|
|
continue
|
||
|
|
if fan_obj.air_flow != self.board_air_flow:
|
||
|
|
fanairflow_debug("%s air flow error, origin name: [%s], display name: [%s], fan air flow [%s], board air flow [%s]" %
|
||
|
|
(fan_obj.name, fan_obj.origin_name, fan_obj.display_name, fan_obj.air_flow, self.board_air_flow))
|
||
|
|
air_flow_inconsistent_flag_tmp = True
|
||
|
|
fan_obj.air_flow_inconsistent = True
|
||
|
|
else:
|
||
|
|
fanairflow_debug("%s air flow check ok, origin name: [%s], display name: [%s], fan air flow: [%s], board air flow: [%s]" %
|
||
|
|
(fan_obj.name, fan_obj.origin_name, fan_obj.display_name, fan_obj.air_flow, self.board_air_flow))
|
||
|
|
fan_obj.air_flow_inconsistent = False
|
||
|
|
self.air_flow_inconsistent_flag = air_flow_inconsistent_flag_tmp
|
||
|
|
else:
|
||
|
|
fanairflow_debug("air flow monitor not open, set air_flow_inconsistent_flag False")
|
||
|
|
self.air_flow_inconsistent_flag = False
|
||
|
|
return
|
||
|
|
|
||
|
|
def do_fancontrol(self):
|
||
|
|
pwm_list = []
|
||
|
|
pwm_min = self.__min_pwm
|
||
|
|
pwm_list.append(pwm_min)
|
||
|
|
|
||
|
|
# first check fan air flow
|
||
|
|
self.check_fan_air_flow()
|
||
|
|
fanairflow_debug("check_fan_air_flow, air_flow_inconsistent_flag: %s" % self.air_flow_inconsistent_flag)
|
||
|
|
# get_monitor_temp
|
||
|
|
self.get_monitor_temp()
|
||
|
|
fancontrol_debug("last_pwm = 0x%x" % self.__pwm)
|
||
|
|
# openloop
|
||
|
|
inlettemp = self.__temps_threshold_config.get(INLET_TEMP)['temp']
|
||
|
|
linear_value = self.openloop.linear_cacl(inlettemp)
|
||
|
|
if linear_value is None:
|
||
|
|
linear_value = self.__min_pwm
|
||
|
|
pwm_list.append(linear_value)
|
||
|
|
fancontrol_debug("linear_value = 0x%x" % linear_value)
|
||
|
|
|
||
|
|
curve_value = self.openloop.curve_cacl(inlettemp)
|
||
|
|
if curve_value is None:
|
||
|
|
curve_value = self.__min_pwm
|
||
|
|
pwm_list.append(curve_value)
|
||
|
|
fancontrol_debug("curve_value = 0x%x" % curve_value)
|
||
|
|
|
||
|
|
# hyst
|
||
|
|
for hyst_index in self.__hyst_config.values():
|
||
|
|
temp_name = hyst_index['name']
|
||
|
|
tmp_temp = int(self.__temps_threshold_config.get(temp_name)['temp']) # make sure temp is int
|
||
|
|
hyst_value = self.hyst.cacl(temp_name, tmp_temp)
|
||
|
|
if hyst_value is None:
|
||
|
|
hyst_value = self.__min_pwm
|
||
|
|
pwm_list.append(hyst_value)
|
||
|
|
fancontrol_debug("%s hyst_value = 0x%x" % (temp_name, hyst_value))
|
||
|
|
|
||
|
|
# pid
|
||
|
|
for pid_index in self.__pid_config.values():
|
||
|
|
temp_name = pid_index['name']
|
||
|
|
tmp_temp = self.__temps_threshold_config.get(temp_name)['temp']
|
||
|
|
if tmp_temp is not None:
|
||
|
|
tmp_temp = int(tmp_temp) # make sure temp is int
|
||
|
|
invalid_temp = self.__temps_threshold_config.get(temp_name)['invalid']
|
||
|
|
error_temp = self.__temps_threshold_config.get(temp_name)['error']
|
||
|
|
if tmp_temp == invalid_temp: # temp is invalid
|
||
|
|
temp = None
|
||
|
|
self.pid.cacl(self.__pwm, temp_name, temp) # temp invalid, PID need to record None
|
||
|
|
pid_value = self.__temp_invalid_pid_pwm
|
||
|
|
fancontrol_debug("%s is invalid, pid_value = 0x%x" % (temp_name, pid_value))
|
||
|
|
fancontrol_debug("temp = %d, invalid_temp = %d" % (tmp_temp, invalid_temp))
|
||
|
|
elif tmp_temp == error_temp: # temp is error
|
||
|
|
temp = None
|
||
|
|
self.pid.cacl(self.__pwm, temp_name, temp) # temp error, PID need to record None
|
||
|
|
pid_value = self.__temp_error_pid_pwm
|
||
|
|
fancontrol_debug("%s is error, pid_value = 0x%x" % (temp_name, pid_value))
|
||
|
|
fancontrol_debug("temp = %d, error_temp = %d" % (tmp_temp, error_temp))
|
||
|
|
else:
|
||
|
|
pid_value = self.pid.cacl(self.__pwm, temp_name, tmp_temp)
|
||
|
|
else: # temp get failed
|
||
|
|
pid_value = self.pid.cacl(self.__pwm, temp_name, tmp_temp)
|
||
|
|
if pid_value is None:
|
||
|
|
pid_value = self.__min_pwm
|
||
|
|
pwm_list.append(pid_value)
|
||
|
|
fancontrol_debug("%s pid_value = 0x%x" % (temp_name, pid_value))
|
||
|
|
|
||
|
|
# abnormal
|
||
|
|
abnormal_value = self.abnormal_check()
|
||
|
|
pwm_list.append(abnormal_value)
|
||
|
|
fancontrol_debug("abnormal_value = 0x%x" % abnormal_value)
|
||
|
|
|
||
|
|
if self.__fan_plug_in_countdown > 0 and self.__fan_absent_num == 0:
|
||
|
|
fancontrol_debug("fan plug in countdown %d, set plug in pwm: 0x%x" %
|
||
|
|
(self.__fan_plug_in_countdown, self.__fan_plug_in_pwm))
|
||
|
|
self.__pwm = self.__fan_plug_in_pwm
|
||
|
|
self.__fan_plug_in_countdown -= 1
|
||
|
|
else:
|
||
|
|
self.__pwm = max(pwm_list)
|
||
|
|
fancontrol_debug("__pwm = 0x%x\n" % self.__pwm)
|
||
|
|
if self.air_flow_inconsistent_flag is True:
|
||
|
|
fanairflow_debug("air flow inconsistent, set all fan speed pwm")
|
||
|
|
self.set_all_fan_speed_pwm(self.__pwm)
|
||
|
|
else:
|
||
|
|
fanairflow_debug("air flow consistent, deal fan error policy")
|
||
|
|
fan_pwm_dict = self.get_fan_pwm_dict(self.__pwm)
|
||
|
|
self.set_fan_pwm_independent(fan_pwm_dict, self.__pwm)
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
start_time = time.time()
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
debug_init()
|
||
|
|
if self.__fan_status_interval > 0 and self.__fan_status_interval < self.__interval:
|
||
|
|
delta_time = time.time() - start_time
|
||
|
|
if delta_time >= self.__interval or delta_time < 0:
|
||
|
|
self.do_fancontrol()
|
||
|
|
start_time = time.time()
|
||
|
|
else:
|
||
|
|
self.checkFanPresence()
|
||
|
|
time.sleep(self.__fan_status_interval)
|
||
|
|
else:
|
||
|
|
self.do_fancontrol()
|
||
|
|
time.sleep(self.__interval)
|
||
|
|
except Exception as e:
|
||
|
|
traceback.print_exc()
|
||
|
|
fancontrol_error(str(e))
|
||
|
|
|
||
|
|
def set_all_fan_speed_pwm(self, pwm):
|
||
|
|
fan_pwm_dict = {}
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
fan_pwm_dict[fan_name] = pwm
|
||
|
|
self.set_fan_pwm_independent(fan_pwm_dict, pwm)
|
||
|
|
|
||
|
|
def set_fan_pwm_independent(self, fan_pwm_dict, psu_fan_pwm):
|
||
|
|
if self.air_flow_inconsistent_flag is True:
|
||
|
|
psu_fan_pwm = self.air_flow_error_psu_pwm
|
||
|
|
fancontrol_debug("air flow inconsistent, set psu fan pwm: 0x%x" % psu_fan_pwm)
|
||
|
|
for fan_obj in self.fan_obj_list:
|
||
|
|
if fan_obj.air_flow_inconsistent is True:
|
||
|
|
fan_pwm_dict[fan_obj.name] = self.air_flow_error_fan_pwm
|
||
|
|
fanairflow_debug("%s air flow error, origin name: [%s], display name: [%s], fan air flow: [%s], board air flow: [%s], set fan pwm: 0x%x" %
|
||
|
|
(fan_obj.name, fan_obj.origin_name, fan_obj.display_name, fan_obj.air_flow, self.board_air_flow, self.air_flow_error_fan_pwm))
|
||
|
|
else:
|
||
|
|
fan_pwm_dict[fan_obj.name] = self.air_flow_correct_fan_pwm
|
||
|
|
fanairflow_debug("%s air flow correct, origin name: [%s], display name: [%s], fan air flow: [%s], board air flow: [%s], set fan pwm: 0x%x" %
|
||
|
|
(fan_obj.name, fan_obj.origin_name, fan_obj.display_name, fan_obj.air_flow, self.board_air_flow, self.air_flow_correct_fan_pwm))
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
self.fan_set_speed_pwm_by_name(fan_name, fan_pwm_dict[fan_name])
|
||
|
|
if self.__psu_fan_control == 1:
|
||
|
|
self.set_psu_fan_speed_pwm(psu_fan_pwm)
|
||
|
|
|
||
|
|
def get_fans(self):
|
||
|
|
return self.int_case.get_fans()
|
||
|
|
|
||
|
|
def get_speed(self, fan_name, rotor_index):
|
||
|
|
return self.int_case.get_fan_speed(fan_name, rotor_index)
|
||
|
|
|
||
|
|
def get_speed_pwm(self, fan_name, rotor_index):
|
||
|
|
return self.int_case.get_fan_speed_pwm(fan_name, rotor_index)
|
||
|
|
|
||
|
|
def fan_get_speed_pwm(self):
|
||
|
|
fans = self.get_fans()
|
||
|
|
for fan in fans:
|
||
|
|
rotor_len = self.get_rotor_number(fan.name)
|
||
|
|
for i in range(rotor_len):
|
||
|
|
fancontrol_debug("%s rotor%d: %d" % (fan.name, i + 1, self.get_speed_pwm(fan.name, i + 1)))
|
||
|
|
|
||
|
|
def fan_set_speed_pwm_by_name(self, fan_name, pwm):
|
||
|
|
duty = round(pwm * 100 / 255)
|
||
|
|
rotor_len = self.get_rotor_number(fan_name)
|
||
|
|
for i in range(rotor_len):
|
||
|
|
val = self.int_case.set_fan_speed_pwm(fan_name, i + 1, duty)
|
||
|
|
if val != 0:
|
||
|
|
fancontrol_error("%s rotor%d: %d" % (fan_name, i + 1, val))
|
||
|
|
|
||
|
|
def set_psu_fan_speed_pwm(self, pwm):
|
||
|
|
duty = round(pwm * 100 / 255)
|
||
|
|
psu_num = self.get_psu_total_number()
|
||
|
|
for i in range(psu_num):
|
||
|
|
psu_name = "PSU" + str(i + 1)
|
||
|
|
status = self.int_case.set_psu_fan_speed_pwm(psu_name, int(duty))
|
||
|
|
if status != True:
|
||
|
|
fancontrol_error("set %s speed fail" % psu_name)
|
||
|
|
|
||
|
|
def fan_obj_init(self):
|
||
|
|
fan_num = self.get_fan_total_number()
|
||
|
|
for i in range(fan_num):
|
||
|
|
fan_name = "FAN" + str(i + 1)
|
||
|
|
fan_obj = DevFan(fan_name, self.int_case)
|
||
|
|
self.fan_obj_list.append(fan_obj)
|
||
|
|
fanairflow_debug("fan object initialize success")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
debug_init()
|
||
|
|
fancontrol_debug("enter main")
|
||
|
|
fan_control = fancontrol()
|
||
|
|
fan_control.fan_obj_init()
|
||
|
|
fan_control.run()
|