[yang-models]: Remove PLY Extensions and change translation code. (#6915)
* [yang-models]: Remove PLY Extensions and change translation code. With assumption that TABLE_SEPARATOR and ENTRY_SEPARATOR for configDB is always "|", translation from configDB.json to sonicYang.json can be done based on keys specified in YANG Lists inside YANG models. So removing extensions is good idea. Changes: -- Remove use of regex in Translation code. -- Remove regex Extensions from YANG models. -- Improved debugging i.e. log on stdout in case of any Exception from sonic-yang-mgmt, so that failed tests can be debugged faster. Also this is good to debug Dynamic port breakout issues. -- Minor Test changes. Co-authored-by: lguohan <lguohan@gmail.com>
This commit is contained in:
Praveen Chaudhary
GitHub
parent
6d23a78ffb
commit
d3d0c2623d
@ -47,11 +47,13 @@ class SonicYang(SonicYangExtMixin):
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
def sysLog(self, debug=syslog.LOG_INFO, msg=None):
|
||||
def sysLog(self, debug=syslog.LOG_INFO, msg=None, doPrint=False):
|
||||
|
||||
# log debug only if enabled
|
||||
if self.DEBUG == False and debug == syslog.LOG_DEBUG:
|
||||
return
|
||||
if doPrint:
|
||||
print("{}({}):{}".format(self.SYSLOG_IDENTIFIER, debug, msg))
|
||||
syslog.openlog(self.SYSLOG_IDENTIFIER)
|
||||
syslog.syslog(debug, msg)
|
||||
syslog.closelog()
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
|
||||
from __future__ import print_function
|
||||
import yang as ly
|
||||
import re
|
||||
import syslog
|
||||
|
||||
from json import dump, dumps, loads
|
||||
@ -47,7 +46,8 @@ class SonicYangExtMixin:
|
||||
self._createDBTableToModuleMap()
|
||||
|
||||
except Exception as e:
|
||||
print("Yang Models Load failed")
|
||||
self.sysLog(msg="Yang Models Load failed:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("Yang Models Load failed\n{}".format(str(e)))
|
||||
|
||||
return True
|
||||
@ -65,6 +65,8 @@ class SonicYangExtMixin:
|
||||
self.yJson.append(parse(xml))
|
||||
self.sysLog(msg="Parsed Json for {}".format(m.name()))
|
||||
except Exception as e:
|
||||
self.sysLog(msg="JSON schema Load failed:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise e
|
||||
|
||||
return
|
||||
@ -90,7 +92,9 @@ class SonicYangExtMixin:
|
||||
|
||||
# top level container must exist for rest of the yang files and it should
|
||||
# have same name as module name.
|
||||
assert topLevelContainer['@name'] == moduleName
|
||||
if topLevelContainer['@name'] != moduleName:
|
||||
raise(SonicYangException("topLevelContainer mismatch {}:{}".\
|
||||
format(topLevelContainer['@name'], moduleName)))
|
||||
|
||||
# Each container inside topLevelContainer maps to a sonic config table.
|
||||
container = topLevelContainer['container']
|
||||
@ -154,25 +158,22 @@ class SonicYangExtMixin:
|
||||
Input:
|
||||
tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64"
|
||||
keys: key string from YANG list, i.e. 'vlan_name ip-prefix'.
|
||||
regex: A regex to extract keys from tableKeys, good to have it as accurate as possible.
|
||||
|
||||
Return:
|
||||
KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"}
|
||||
"""
|
||||
def _extractKey(self, tableKey, keys, regex):
|
||||
def _extractKey(self, tableKey, keys):
|
||||
|
||||
keyList = keys.split()
|
||||
# get the value groups
|
||||
value = re.match(regex, tableKey)
|
||||
value = tableKey.split("|")
|
||||
# match lens
|
||||
if len(keyList) != len(value):
|
||||
raise Exception("Value not found for {} in {}".format(keys, tableKey))
|
||||
# create the keyDict
|
||||
i = 1
|
||||
keyDict = dict()
|
||||
for k in keyList:
|
||||
if value.group(i):
|
||||
keyDict[k] = value.group(i)
|
||||
else:
|
||||
raise Exception("Value not found for {} in {}".format(k, tableKey))
|
||||
i = i + 1
|
||||
for i in range(len(keyList)):
|
||||
keyDict[keyList[i]] = value[i].strip()
|
||||
|
||||
return keyDict
|
||||
|
||||
@ -264,22 +265,21 @@ class SonicYangExtMixin:
|
||||
Xlate a list
|
||||
This function will xlate from a dict in config DB to a Yang JSON list
|
||||
using yang model. Output will be go in self.xlateJson
|
||||
|
||||
Note: Exceptions from this function are collected in exceptionList and
|
||||
are displayed only when an entry is not xlated properly from ConfigDB
|
||||
to sonic_yang.json.
|
||||
"""
|
||||
def _xlateList(self, model, yang, config, table):
|
||||
def _xlateList(self, model, yang, config, table, exceptionList):
|
||||
|
||||
#create a dict to map each key under primary key with a dict yang model.
|
||||
#This is done to improve performance of mapping from values of TABLEs in
|
||||
#config DB to leaf in YANG LIST.
|
||||
leafDict = self._createLeafDict(model)
|
||||
|
||||
# fetch regex from YANG models.
|
||||
keyRegEx = model['ext:key-regex-configdb-to-yang']['@value']
|
||||
# seperator `|` has special meaning in regex, so change it appropriately.
|
||||
keyRegEx = re.sub(r'\|', r'\\|', keyRegEx)
|
||||
# get keys from YANG model list itself
|
||||
listKeys = model['key']['@value']
|
||||
self.sysLog(msg="xlateList regex:{} keyList:{}".\
|
||||
format(keyRegEx, listKeys))
|
||||
self.sysLog(msg="xlateList keyList:{}".format(listKeys))
|
||||
|
||||
primaryKeys = list(config.keys())
|
||||
for pkey in primaryKeys:
|
||||
@ -288,7 +288,7 @@ class SonicYangExtMixin:
|
||||
self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\
|
||||
format(pkey))
|
||||
# Find and extracts key from each dict in config
|
||||
keyDict = self._extractKey(pkey, listKeys, keyRegEx)
|
||||
keyDict = self._extractKey(pkey, listKeys)
|
||||
# fill rest of the values in keyDict
|
||||
for vKey in config[pkey]:
|
||||
self.sysLog(syslog.LOG_DEBUG, "xlateList vkey {}".format(vKey))
|
||||
@ -300,7 +300,9 @@ class SonicYangExtMixin:
|
||||
|
||||
except Exception as e:
|
||||
# log debug, because this exception may occur with multilists
|
||||
self.sysLog(syslog.LOG_DEBUG, "xlateList Exception {}".format(e))
|
||||
self.sysLog(msg="xlateList Exception:{}".format(str(e)), \
|
||||
debug=syslog.LOG_DEBUG, doPrint=True)
|
||||
exceptionList.append(str(e))
|
||||
# with multilist, we continue matching other keys.
|
||||
continue
|
||||
|
||||
@ -310,12 +312,12 @@ class SonicYangExtMixin:
|
||||
Process list inside a Container.
|
||||
This function will call xlateList based on list(s) present in Container.
|
||||
"""
|
||||
def _xlateListInContainer(self, model, yang, configC, table):
|
||||
def _xlateListInContainer(self, model, yang, configC, table, exceptionList):
|
||||
clist = model
|
||||
#print(clist['@name'])
|
||||
yang[clist['@name']] = list()
|
||||
self.sysLog(msg="xlateProcessListOfContainer: {}".format(clist['@name']))
|
||||
self._xlateList(clist, yang[clist['@name']], configC, table)
|
||||
self._xlateList(clist, yang[clist['@name']], configC, table, exceptionList)
|
||||
# clean empty lists
|
||||
if len(yang[clist['@name']]) == 0:
|
||||
del yang[clist['@name']]
|
||||
@ -354,16 +356,18 @@ class SonicYangExtMixin:
|
||||
# To Handle multiple Lists, Make a copy of config, because we delete keys
|
||||
# from config after each match. This is done to match one pkey with one list.
|
||||
configC = config.copy()
|
||||
|
||||
exceptionList = list()
|
||||
clist = model.get('list')
|
||||
# If single list exists in container,
|
||||
if clist and isinstance(clist, dict) and \
|
||||
clist['@name'] == model['@name']+"_LIST" and bool(configC):
|
||||
self._xlateListInContainer(clist, yang, configC, table)
|
||||
self._xlateListInContainer(clist, yang, configC, table, \
|
||||
exceptionList)
|
||||
# If multi-list exists in container,
|
||||
elif clist and isinstance(clist, list) and bool(configC):
|
||||
for modelList in clist:
|
||||
self._xlateListInContainer(modelList, yang, configC, table)
|
||||
self._xlateListInContainer(modelList, yang, configC, table, \
|
||||
exceptionList)
|
||||
|
||||
# Handle container(s) in container
|
||||
ccontainer = model.get('container')
|
||||
@ -388,7 +392,10 @@ class SonicYangExtMixin:
|
||||
|
||||
# All entries in copy of config must have been parsed.
|
||||
if len(configC):
|
||||
self.sysLog(syslog.LOG_ERR, "Alert: Remaining keys in Config")
|
||||
self.sysLog(msg="All Keys are not parsed in {}\n{}".format(table, \
|
||||
configC.keys()), debug=syslog.LOG_ERR, doPrint=True)
|
||||
self.sysLog(msg="exceptionList:{}".format(exceptionList), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise(Exception("All Keys are not parsed in {}\n{}".format(table, \
|
||||
configC.keys())))
|
||||
|
||||
@ -433,22 +440,24 @@ class SonicYangExtMixin:
|
||||
"""
|
||||
create config DB table key from entry in yang JSON
|
||||
"""
|
||||
def _createKey(self, entry, regex):
|
||||
def _createKey(self, entry, keys):
|
||||
|
||||
keyDict = dict()
|
||||
keyV = regex
|
||||
# get the keys from regex of key extractor
|
||||
keyList = re.findall(r'<(.*?)>', regex)
|
||||
keyList = keys.split()
|
||||
keyV = ""
|
||||
|
||||
for key in keyList:
|
||||
val = entry.get(key)
|
||||
if val:
|
||||
#print("pair: {} {}".format(key, val))
|
||||
keyDict[key] = sval = str(val)
|
||||
keyV = re.sub(r'<'+key+'>', sval, keyV)
|
||||
keyV += sval + "|"
|
||||
#print("VAL: {} {}".format(regex, keyV))
|
||||
else:
|
||||
raise Exception("key {} not found in entry".format(key))
|
||||
#print("kDict {}".format(keyDict))
|
||||
keyV = keyV.rstrip("|")
|
||||
|
||||
return keyV, keyDict
|
||||
|
||||
"""
|
||||
@ -478,10 +487,8 @@ class SonicYangExtMixin:
|
||||
"""
|
||||
def _revXlateList(self, model, yang, config, table):
|
||||
|
||||
# fetch regex from YANG models
|
||||
keyRegEx = model['ext:key-regex-yang-to-configdb']['@value']
|
||||
self.sysLog(msg="revXlateList regex:{}".format(keyRegEx))
|
||||
|
||||
# get keys from YANG model list itself
|
||||
listKeys = model['key']['@value']
|
||||
# create a dict to map each key under primary key with a dict yang model.
|
||||
# This is done to improve performance of mapping from values of TABLEs in
|
||||
# config DB to leaf in YANG LIST.
|
||||
@ -491,7 +498,7 @@ class SonicYangExtMixin:
|
||||
if "_LIST" in model['@name']:
|
||||
for entry in yang:
|
||||
# create key of config DB table
|
||||
pkey, pkeydict = self._createKey(entry, keyRegEx)
|
||||
pkey, pkeydict = self._createKey(entry, listKeys)
|
||||
self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey))
|
||||
config[pkey]= dict()
|
||||
# fill rest of the entries
|
||||
@ -631,7 +638,8 @@ class SonicYangExtMixin:
|
||||
list = self._findYangList(container, table+"_LIST")
|
||||
xpath = xpath + "/" + list['key']['@value'].split()[0]
|
||||
except Exception as e:
|
||||
print("find xpath of port Leaf failed")
|
||||
self.sysLog(msg="find xpath of port Leaf failed", \
|
||||
debug = syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("find xpath of port Leaf failed\n{}".format(str(e)))
|
||||
|
||||
return xpath
|
||||
@ -649,7 +657,8 @@ class SonicYangExtMixin:
|
||||
list = self._findYangList(container, table+"_LIST")
|
||||
xpath = self._findXpathList(xpath, list, [portName])
|
||||
except Exception as e:
|
||||
print("find xpath of port failed")
|
||||
self.sysLog(msg="find xpath of port failed", \
|
||||
debug = syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("find xpath of port failed\n{}".format(str(e)))
|
||||
|
||||
return xpath
|
||||
@ -671,6 +680,8 @@ class SonicYangExtMixin:
|
||||
xpath = xpath + '['+listKey+'=\''+keys[i]+'\']'
|
||||
i = i + 1
|
||||
except Exception as e:
|
||||
self.sysLog(msg="_findXpathList:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise e
|
||||
|
||||
return xpath
|
||||
@ -703,7 +714,8 @@ class SonicYangExtMixin:
|
||||
|
||||
except Exception as e:
|
||||
self.root = None
|
||||
print("Data Loading Failed")
|
||||
self.sysLog(msg="Data Loading Failed:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("Data Loading Failed\n{}".format(str(e)))
|
||||
|
||||
return True
|
||||
@ -725,7 +737,8 @@ class SonicYangExtMixin:
|
||||
self._revXlateConfigDB(revXlateFile=revXlateFile)
|
||||
|
||||
except Exception as e:
|
||||
print("Get Data Tree Failed")
|
||||
self.sysLog(msg="Get Data Tree Failed:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("Get Data Tree Failed\n{}".format(str(e)))
|
||||
|
||||
return self.revXlateJson
|
||||
@ -760,6 +773,8 @@ class SonicYangExtMixin:
|
||||
if self._deleteNode(xpath=xpath, node=node) == False:
|
||||
raise Exception('_deleteNode failed')
|
||||
except Exception as e:
|
||||
self.sysLog(msg="deleteNode:{}".format(str(e)), \
|
||||
debug=syslog.LOG_ERR, doPrint=True)
|
||||
raise SonicYangException("Failed to delete node {}\n{}".\
|
||||
format( xpath, str(e)))
|
||||
|
||||
|
||||
@ -340,6 +340,11 @@ class Test_SonicYang(object):
|
||||
print("Xlate and Rev Xlate Passed")
|
||||
else:
|
||||
print("Xlate and Rev Xlate failed")
|
||||
# print for better debugging, in case of failure.
|
||||
print("syc.jIn: {}".format({t:syc.jIn[t].keys() \
|
||||
for t in syc.jIn.keys()}))
|
||||
print("syc.revXlateJson: {}".format({t:syc.revXlateJson[t].keys() \
|
||||
for t in syc.revXlateJson.keys()}))
|
||||
# make it fail
|
||||
assert False == True
|
||||
|
||||
|
||||
@ -45,10 +45,6 @@ module sonic-acl {
|
||||
|
||||
key "ACL_TABLE_NAME RULE_NAME";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)|([a-zA-Z0-9_-]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<ACL_TABLE_NAME>|<RULE_NAME>";
|
||||
|
||||
leaf ACL_TABLE_NAME {
|
||||
type leafref {
|
||||
path "/acl:sonic-acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME";
|
||||
@ -233,10 +229,6 @@ module sonic-acl {
|
||||
|
||||
key "ACL_TABLE_NAME";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<ACL_TABLE_NAME>";
|
||||
|
||||
leaf ACL_TABLE_NAME {
|
||||
type string;
|
||||
}
|
||||
|
||||
@ -26,10 +26,6 @@ module sonic-breakout_cfg {
|
||||
|
||||
key "port";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<port>";
|
||||
|
||||
/*
|
||||
* This is leaf instead of a leafref towards port, because this
|
||||
* table will contains Parent ports and their current breakout-mode,
|
||||
|
||||
@ -35,10 +35,6 @@ module sonic-device_neighbor {
|
||||
|
||||
key "peer_name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<peer_name>";
|
||||
|
||||
leaf peer_name {
|
||||
type string {
|
||||
length 1..255;
|
||||
|
||||
@ -10,14 +10,4 @@ module sonic-extension {
|
||||
revision 2019-07-01 {
|
||||
description "First Revision";
|
||||
}
|
||||
|
||||
extension key-regex-configdb-to-yang {
|
||||
description "Key regex used to convert config DB keys to YANG Config";
|
||||
argument "value";
|
||||
}
|
||||
|
||||
extension key-regex-yang-to-configdb {
|
||||
description "Key regex used to convert config DB keys to YANG Config";
|
||||
argument "value";
|
||||
}
|
||||
}
|
||||
|
||||
@ -38,10 +38,6 @@ module sonic-interface {
|
||||
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
leaf name {
|
||||
type leafref {
|
||||
path /port:sonic-port/port:PORT/port:PORT_LIST/port:name;
|
||||
@ -63,10 +59,6 @@ module sonic-interface {
|
||||
|
||||
key "name ip-prefix";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)|([a-fA-F0-9:./]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>|<ip-prefix>";
|
||||
|
||||
leaf name {
|
||||
/* This node must be present in INTERFACE_LIST */
|
||||
must "(current() = ../../INTERFACE_LIST[name=current()]/name)"
|
||||
|
||||
@ -29,10 +29,6 @@ module sonic-loopback-interface {
|
||||
list LOOPBACK_INTERFACE_LIST {
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
leaf name{
|
||||
type string;
|
||||
}
|
||||
@ -50,10 +46,6 @@ module sonic-loopback-interface {
|
||||
|
||||
key "name ip-prefix";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)|([a-fA-F0-9:./]+$)";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>|<ip-prefix>";
|
||||
|
||||
leaf name{
|
||||
/* This node must be present in LOOPBACK_INTERFACE_LIST */
|
||||
must "(current() = ../../LOOPBACK_INTERFACE_LIST[name=current()]/name)"
|
||||
|
||||
@ -31,10 +31,6 @@ module sonic-port{
|
||||
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
leaf name {
|
||||
type string {
|
||||
length 1..128;
|
||||
|
||||
@ -35,10 +35,6 @@ module sonic-portchannel {
|
||||
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(PortChannel[0-9]{1,4})$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
leaf name {
|
||||
type string {
|
||||
length 1..128;
|
||||
|
||||
@ -42,10 +42,6 @@ module sonic-vlan {
|
||||
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
leaf name {
|
||||
type leafref {
|
||||
path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name;
|
||||
@ -65,10 +61,6 @@ module sonic-vlan {
|
||||
|
||||
key "name ip-prefix";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)|([a-fA-F0-9:./]+$)";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>|<ip-prefix>";
|
||||
|
||||
leaf name {
|
||||
/* This node must be present in VLAN_INTERFACE_LIST */
|
||||
must "(current() = ../../VLAN_INTERFACE_LIST[name=current()]/name)"
|
||||
@ -119,11 +111,7 @@ module sonic-vlan {
|
||||
list VLAN_LIST {
|
||||
|
||||
key "name";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>";
|
||||
|
||||
|
||||
leaf name {
|
||||
type string {
|
||||
pattern 'Vlan([0-9]{1,3}|[1-3][0-9]{3}|[4][0][0-8][0-9]|[4][0][9][0-4])';
|
||||
@ -168,10 +156,6 @@ module sonic-vlan {
|
||||
|
||||
key "name port";
|
||||
|
||||
ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9-_]+)|(Ethernet[0-9]+)$";
|
||||
|
||||
ext:key-regex-yang-to-configdb "<name>|<port>";
|
||||
|
||||
leaf name {
|
||||
type leafref {
|
||||
path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user