matthew/sonic-buildimage
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

handle json load exception in bgpmon (#17856)

Browse Source 
Why I did it
ICM reported due to "BGPMon Process exited" which was caused by json load exception.

Work item tracking
Microsoft ADO (number only):
25916773
How I did it
Add an exception handle during json load.

How to verify it
Verified locally, add debug log to modify the output string of cmd to make it not with json formation, then check the syslog.
This commit is contained in:
Liping Xu 2024-01-29 15:55:28 +08:00 committed by mssonicbld
parent 8ef47bf855
commit 5d43f7c511
1 changed files with 38 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/sonic-bgpcfgd/bgpmon/bgpmon.py
View File

@ -49,6 +49,7 @@ class BgpStateGet:
self.db.connect(self.db.STATE_DB, False) self.db.connect(self.db.STATE_DB, False)
self.pipe = swsscommon.RedisPipeline(self.db.get_redis_client(self.db.STATE_DB)) self.pipe = swsscommon.RedisPipeline(self.db.get_redis_client(self.db.STATE_DB))
self.db.delete_all_by_pattern(self.db.STATE_DB, "NEIGH_STATE_TABLE|*" ) self.db.delete_all_by_pattern(self.db.STATE_DB, "NEIGH_STATE_TABLE|*" )
self.MAX_RETRY_ATTEMPTS = 3
# A quick way to check if there are anything happening within BGP is to # A quick way to check if there are anything happening within BGP is to
# check its log file has any activities. This is by checking its modified # check its log file has any activities. This is by checking its modified
@ -77,10 +78,17 @@ class BgpStateGet:
# Get a new snapshot of BGP neighbors and store them in the "new" location # Get a new snapshot of BGP neighbors and store them in the "new" location
def get_all_neigh_states(self): def get_all_neigh_states(self):
cmd = ["vtysh", "-c", 'show bgp summary json'] cmd = ["vtysh", "-c", 'show bgp summary json']
retry_attempt = 0
while retry_attempt < self.MAX_RETRY_ATTEMPTS:
try:
rc, output = getstatusoutput_noshell(cmd) rc, output = getstatusoutput_noshell(cmd)
if rc: if rc:
syslog.syslog(syslog.LOG_ERR, "*ERROR* Failed with rc:{} when execute: {}".format(rc, cmd)) syslog.syslog(syslog.LOG_ERR, "*ERROR* Failed with rc:{} when execute: {}".format(rc, cmd))
return return
if len(output) == 0:
syslog.syslog(syslog.LOG_WARNING, "*WARNING* output none when execute: {}".format(cmd))
return
peer_info = json.loads(output) peer_info = json.loads(output)
# cmd ran successfully, safe to Clean the "new" set/dict for new snapshot # cmd ran successfully, safe to Clean the "new" set/dict for new snapshot
@ -89,6 +97,25 @@ class BgpStateGet:
for key, value in peer_info.items(): for key, value in peer_info.items():
if key == "ipv4Unicast" or key == "ipv6Unicast": if key == "ipv4Unicast" or key == "ipv6Unicast":
self.update_new_peer_states(value) self.update_new_peer_states(value)
return
except json.JSONDecodeError as decode_error:
# Log the exception and retry if within the maximum attempts
retry_attempt += 1
syslog.syslog(syslog.LOG_WARNING, "*WARNING* JSONDecodeError: {} when execute: {} Retry attempt: {}".format(decode_error, cmd, retry_attempt))
time.sleep(1)
continue
except Exception as e:
# Log other exceptions and return failure
retry_attempt += 1
syslog.syslog(syslog.LOG_WARNING, "*WARNING* An unexpected error occurred: {} when execute: {} Retry attempt: {}".format(e, cmd, retry_attempt))
time.sleep(1)
continue
# Log an error if the maximum retry attempts are reached
syslog.syslog(syslog.LOG_ERR, "*ERROR* Maximum retry attempts reached. Failed to execute: {} Output: {}".format(cmd, output))
sys.exit(1)
# This method will take the caller's dictionary which contains the peer state operation # This method will take the caller's dictionary which contains the peer state operation
# That need to be updated in StateDB using Redis pipeline. # That need to be updated in StateDB using Redis pipeline.