matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Mellanox] Fix issue: watchdogutil command does not work (#16091)

Browse Source 
- Why I did it
watchdogutil uses platform API watchdog instance to control/query watchdog status. In Nvidia watchdog status, it caches "armed" status in a object member "WatchdogImplBase.armed". This is not working for CLI infrastructure because each CLI will create a new watchdog instance, the status cached in previous instance will totally lose. Consider following commands:

admin@sonic:~$ sudo watchdogutil arm -s 100      =====> watchdog instance1, armed=True
Watchdog armed for 100 seconds
admin@sonic:~$ sudo watchdogutil status             ======> watchdog instance2, armed=False
Status: Unarmed
admin@sonic:~$ sudo watchdogutil disarm            =======> watchdog instance3, armed=False
Failed to disarm Watchdog

- How I did it
Use sysfs to query watchdog status

- How to verify it
Manual test
Unit test
This commit is contained in:
Junchao-Mellanox 2023-08-23 14:30:58 +08:00 committed by mssonicbld
parent dc3991309b
commit d13341fd9b
2 changed files with 50 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
platform/mellanox/mlnx-platform-api/sonic_platform/watchdog.py
View File

@ -27,6 +27,7 @@ import array
import time
from sonic_platform_base.watchdog_base import WatchdogBase
from . import utils
""" ioctl constants """
IO_WRITE = 0x40000000
@ -80,16 +81,15 @@ class WatchdogImplBase(WatchdogBase):
super(WatchdogImplBase, self).__init__()
self.watchdog_path = wd_device_path
self.watchdog = self.open_handle()
# Opening a watchdog descriptor starts
# watchdog timer;
# by default it should be stopped
self._disablecard()
self.armed = False
self._watchdog = None
self.timeout = self._gettimeout()
@property
def watchdog(self):
if self._watchdog is None:
self._watchdog = self.open_handle()
return self._watchdog
def open_handle(self):
return os.open(self.watchdog_path, os.O_WRONLY)
@ -134,10 +134,7 @@ class WatchdogImplBase(WatchdogBase):
@return watchdog timeout
"""
req = array.array('I', [0])
fcntl.ioctl(self.watchdog, WDIOC_GETTIMEOUT, req, True)
return int(req[0])
return utils.read_int_from_file('/run/hw-management/watchdog/main/timeout')
def _gettimeleft(self):
"""
@ -145,10 +142,7 @@ class WatchdogImplBase(WatchdogBase):
@return time left in seconds
"""
req = array.array('I', [0])
fcntl.ioctl(self.watchdog, WDIOC_GETTIMELEFT, req, True)
return int(req[0])
return utils.read_int_from_file('/run/hw-management/watchdog/main/timeleft')
def arm(self, seconds):
"""
@ -162,11 +156,10 @@ class WatchdogImplBase(WatchdogBase):
try:
if self.timeout != seconds:
self.timeout = self._settimeout(seconds)
if self.armed:
if self.is_armed():
self._keepalive()
else:
self._enablecard()
self.armed = True
ret = self.timeout
except IOError:
pass
@ -179,10 +172,9 @@ class WatchdogImplBase(WatchdogBase):
"""
disarmed = False
if self.armed:
if self.is_armed():
try:
self._disablecard()
self.armed = False
disarmed = True
except IOError:
pass
@ -194,7 +186,7 @@ class WatchdogImplBase(WatchdogBase):
Implements is_armed WatchdogBase API
"""
return self.armed
return utils.read_str_from_file('/run/hw-management/watchdog/main/state') == 'active'
def get_remaining_time(self):
"""
@ -203,7 +195,7 @@ class WatchdogImplBase(WatchdogBase):
timeleft = WD_COMMON_ERROR
if self.armed:
if self.is_armed():
try:
timeleft = self._gettimeleft()
except IOError:
@ -216,13 +208,15 @@ class WatchdogImplBase(WatchdogBase):
Close watchdog
"""
os.close(self.watchdog)
if self._watchdog is not None:
os.close(self._watchdog)
class WatchdogType1(WatchdogImplBase):
"""
Watchdog type 1
"""
TIMESTAMP_FILE = '/tmp/nvidia/watchdog_timestamp'
def arm(self, seconds):
"""
@ -233,7 +227,8 @@ class WatchdogType1(WatchdogImplBase):
ret = WatchdogImplBase.arm(self, seconds)
# Save the watchdog arm timestamp
# requiered for get_remaining_time()
self.arm_timestamp = time.time()
os.makedirs('/tmp/nvidia', exist_ok=True)
utils.write_file(self.TIMESTAMP_FILE, str(time.time()))
return ret
@ -246,8 +241,9 @@ class WatchdogType1(WatchdogImplBase):
timeleft = WD_COMMON_ERROR
if self.armed:
timeleft = int(self.timeout - (time.time() - self.arm_timestamp))
if self.is_armed():
arm_timestamp = utils.read_float_from_file(self.TIMESTAMP_FILE)
timeleft = int(self.timeout - (time.time() - arm_timestamp))
return timeleft

44
platform/mellanox/mlnx-platform-api/tests/test_watchdog.py
View File

@ -84,46 +84,58 @@ class TestWatchdog:
mock_exists.return_value = test_para
assert is_wd_type2('') is test_para
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.open_handle', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.fcntl.ioctl', mock.MagicMock())
def test_arm_disarm_watchdog2(self):
@mock.patch('sonic_platform.utils.read_str_from_file')
def test_is_armed(self, mock_read):
watchdog = WatchdogType2('watchdog2')
assert watchdog.arm(-1) == -1
mock_read.return_value = 'inactive'
assert not watchdog.is_armed()
watchdog.arm(10)
mock_read.return_value = 'active'
assert watchdog.is_armed()
watchdog.arm(5)
assert watchdog.is_armed()
watchdog.disarm()
assert not watchdog.is_armed()
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.open_handle', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.fcntl.ioctl', mock.MagicMock())
def test_arm_disarm_watchdog1(self):
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.is_armed')
def test_arm_disarm_watchdog2(self, mock_is_armed):
watchdog = WatchdogType2('watchdog2')
assert watchdog.arm(-1) == -1
mock_is_armed.return_value = False
watchdog.arm(10)
mock_is_armed.return_value = True
watchdog.arm(5)
watchdog.disarm()
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.open_handle', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.fcntl.ioctl', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.is_armed')
def test_arm_disarm_watchdog1(self, mock_is_armed):
watchdog = WatchdogType1('watchdog1')
assert watchdog.arm(-1) == -1
assert not watchdog.is_armed()
mock_is_armed.return_value = False
watchdog.arm(10)
assert watchdog.is_armed()
mock_is_armed.return_value = True
watchdog.arm(5)
assert watchdog.is_armed()
watchdog.disarm()
assert not watchdog.is_armed()
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.open_handle', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.fcntl.ioctl', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.WatchdogImplBase._gettimeleft', mock.MagicMock(return_value=10))
def test_get_remaining_time_watchdog2(self):
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.is_armed')
def test_get_remaining_time_watchdog2(self, mock_is_armed):
watchdog = WatchdogType2('watchdog2')
mock_is_armed.return_value = False
assert watchdog.get_remaining_time() == -1
watchdog.arm(10)
mock_is_armed.return_value = True
assert watchdog.get_remaining_time() == 10
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.open_handle', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.fcntl.ioctl', mock.MagicMock())
@mock.patch('sonic_platform.watchdog.WatchdogImplBase._gettimeleft', mock.MagicMock(return_value=10))
def test_get_remaining_time_watchdog1(self):
@mock.patch('sonic_platform.watchdog.WatchdogImplBase.is_armed')
def test_get_remaining_time_watchdog1(self, mock_is_armed):
watchdog = WatchdogType1('watchdog2')
mock_is_armed.return_value = False
assert watchdog.get_remaining_time() == -1
watchdog.arm(10)
mock_is_armed.return_value = True
assert watchdog.get_remaining_time() > 0