[YANG MGMT]: Support Grouping translation in YANG Models. (#8318)

Changes:
 -- pre Process Grouping section from all yang models, so it can be used
from any yang model.
-- add jsondiff in setup.py, it is useful for test debugging in case of
failures.
-- use 'stypes' instead of head.
-- pass config DB table name in _createLeafDict().
-- added test config for grouping.
-- white spaces changes.

Note: Changes are done in the way that we can add support for other
Generic YANG statement easily for translation.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com
This commit is contained in:
Praveen Chaudhary 2021-09-30 12:53:34 -07:00 committed by GitHub
parent 1e35915dcf
commit 83108d9c9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 175 additions and 23 deletions

View File

@ -26,12 +26,14 @@ setup(
long_description=readme + '\n\n', long_description=readme + '\n\n',
install_requires = [ install_requires = [
'xmltodict==0.12.0', 'xmltodict==0.12.0',
'ijson==2.6.1' 'ijson==2.6.1',
'jsondiff>=1.2.0',
], ],
tests_require = [ tests_require = [
'pytest>3', 'pytest>3',
'xmltodict==0.12.0', 'xmltodict==0.12.0',
'ijson==2.6.1' 'ijson==2.6.1',
'jsondiff>=1.2.0'
], ],
setup_requires = [ setup_requires = [
'pytest-runner', 'pytest-runner',

View File

@ -36,6 +36,9 @@ class SonicYang(SonicYangExtMixin):
self.revXlateJson = dict() self.revXlateJson = dict()
# below dict store the input config tables which have no YANG models # below dict store the input config tables which have no YANG models
self.tablesWithOutYang = dict() self.tablesWithOutYang = dict()
# below dict will store preProcessed yang objects, which may be needed by
# all yang modules, such as grouping.
self.preProcessedYang = dict()
try: try:
self.ctx = ly.Context(yang_dir) self.ctx = ly.Context(yang_dir)

View File

@ -44,7 +44,6 @@ class SonicYangExtMixin:
self._loadJsonYangModel() self._loadJsonYangModel()
# create a map from config DB table to yang container # create a map from config DB table to yang container
self._createDBTableToModuleMap() self._createDBTableToModuleMap()
except Exception as e: except Exception as e:
self.sysLog(msg="Yang Models Load failed:{}".format(str(e)), \ self.sysLog(msg="Yang Models Load failed:{}".format(str(e)), \
debug=syslog.LOG_ERR, doPrint=True) debug=syslog.LOG_ERR, doPrint=True)
@ -71,6 +70,70 @@ class SonicYangExtMixin:
return return
def _preProcessYangGrouping(self, moduleName, module):
'''
PreProcess Grouping Section of YANG models, and store it in
self.preProcessedYang['grouping'] as
{'<moduleName>':
{'<groupingName>':
[<List of Leafs>]
}
}
Parameters:
moduleName (str): name of yang module.
module (dict): json format of yang module.
Returns:
void
'''
try:
# create grouping dict
if self.preProcessedYang.get('grouping') is None:
self.preProcessedYang['grouping'] = dict()
self.preProcessedYang['grouping'][moduleName] = dict()
# get groupings from yang module
groupings = module['grouping']
# if grouping is a dict, make it a list for common processing
if isinstance(groupings, dict):
groupings = [groupings]
for grouping in groupings:
gName = grouping["@name"]
gLeaf = grouping["leaf"]
self.preProcessedYang['grouping'][moduleName][gName] = gLeaf
except Exception as e:
self.sysLog(msg="_preProcessYangGrouping failed:{}".format(str(e)), \
debug=syslog.LOG_ERR, doPrint=True)
raise e
return
# preProcesss Generic Yang Objects
def _preProcessYang(self, moduleName, module):
'''
PreProcess Generic Section of YANG models by calling
_preProcessYang<SectionName> methods.
Parameters:
moduleName (str): name of yang module.
module (dict): json format of yang module.
Returns:
void
'''
try:
# preProcesss Grouping
if module.get('grouping') is not None:
self._preProcessYangGrouping(moduleName, module)
except Exception as e:
self.sysLog(msg="_preProcessYang failed:{}".format(str(e)), \
debug=syslog.LOG_ERR, doPrint=True)
raise e
return
""" """
Create a map from config DB tables to container in yang model Create a map from config DB tables to container in yang model
This module name and topLevelContainer are fetched considering YANG models are This module name and topLevelContainer are fetched considering YANG models are
@ -82,6 +145,8 @@ class SonicYangExtMixin:
for j in self.yJson: for j in self.yJson:
# get module name # get module name
moduleName = j['module']['@name'] moduleName = j['module']['@name']
# preProcesss Generic Yang Objects
self._preProcessYang(moduleName, j['module'])
# get top level container # get top level container
topLevelContainer = j['module'].get('container') topLevelContainer = j['module'].get('container')
# if top level container is none, this is common yang files, which may # if top level container is none, this is common yang files, which may
@ -104,14 +169,16 @@ class SonicYangExtMixin:
self.confDbYangMap[c['@name']] = { self.confDbYangMap[c['@name']] = {
"module" : moduleName, "module" : moduleName,
"topLevelContainer": topLevelContainer['@name'], "topLevelContainer": topLevelContainer['@name'],
"container": c "container": c,
"yangModule": j['module']
} }
# container is a dict # container is a dict
else: else:
self.confDbYangMap[container['@name']] = { self.confDbYangMap[container['@name']] = {
"module" : moduleName, "module" : moduleName,
"topLevelContainer": topLevelContainer['@name'], "topLevelContainer": topLevelContainer['@name'],
"container": container "container": container,
"yangModule": j['module']
} }
return return
@ -201,13 +268,87 @@ class SonicYangExtMixin:
return return
""" def _findYangModuleFromPrefix(self, prefix, module):
create a dict to map each key under primary key with a dict yang model. '''
This is done to improve performance of mapping from values of TABLEs in Find yang module name from prefix used in given yang module.
config DB to leaf in YANG LIST.
"""
def _createLeafDict(self, model):
Parameters:
prefix (str): prefix used in given yang module.
module (dict): json format of yang module.
Returns:
(str): module name or None
'''
try:
# get imports
yangImports = module.get("import");
if yangImports is None:
return None
# make a list
if isinstance(yangImports, dict):
yangImports = [yangImports]
# find module for given prefix
for yImport in yangImports:
if yImport['prefix']['@value'] == prefix:
return yImport['@module']
except Exception as e:
self.sysLog(msg="_findYangModuleFromPrefix failed:{}".format(str(e)), \
debug=syslog.LOG_ERR, doPrint=True)
raise e
return None
def _fillLeafDictUses(self, uses_s, table, leafDict):
'''
Find the leaf(s) in a grouping which maps to given uses statement,
then fill leafDict with leaf(s) information.
Parameters:
uses_s (str): uses statement in yang module.
table (str): config DB table, this table is being translated.
leafDict (dict): dict with leaf(s) information for List\Container
corresponding to config DB table.
Returns:
(void)
'''
try:
# make a list
if isinstance(uses_s, dict):
uses_s = [uses_s]
# find yang module for current table
table_module = self.confDbYangMap[table]['yangModule']
# uses Example: "@name": "bgpcmn:sonic-bgp-cmn"
for uses in uses_s:
# Assume ':' means reference to another module
if ':' in uses['@name']:
prefix = uses['@name'].split(':')[0].strip()
uses_module = self._findYangModuleFromPrefix(prefix, table_module)
else:
uses_module = table_module
grouping = uses['@name'].split(':')[-1].strip()
leafs = self.preProcessedYang['grouping'][uses_module][grouping]
self._fillLeafDict(leafs, leafDict)
except Exception as e:
self.sysLog(msg="_fillLeafDictUses failed:{}".format(str(e)), \
debug=syslog.LOG_ERR, doPrint=True)
raise e
return
def _createLeafDict(self, model, table):
'''
create a dict to map each key under primary key with a leaf in yang model.
This is done to improve performance of mapping from values of TABLEs in
config DB to leaf in YANG LIST.
Parameters:
module (dict): json format of yang module.
table (str): config DB table, this table is being translated.
Returns:
leafDict (dict): dict with leaf(s) information for List\Container
corresponding to config DB table.
'''
leafDict = dict() leafDict = dict()
#Iterate over leaf, choices and leaf-list. #Iterate over leaf, choices and leaf-list.
self._fillLeafDict(model.get('leaf'), leafDict) self._fillLeafDict(model.get('leaf'), leafDict)
@ -223,6 +364,10 @@ class SonicYangExtMixin:
# leaf-lists # leaf-lists
self._fillLeafDict(model.get('leaf-list'), leafDict, True) self._fillLeafDict(model.get('leaf-list'), leafDict, True)
# uses should map to grouping,
if model.get('uses') is not None:
self._fillLeafDictUses(model.get('uses'), table, leafDict)
return leafDict return leafDict
""" """
@ -245,7 +390,7 @@ class SonicYangExtMixin:
elif 'leafref' in type: elif 'leafref' in type:
vValue = val vValue = val
#TODO: find type in sonic-head, as of now, all are enumeration #TODO: find type in sonic-head, as of now, all are enumeration
elif 'head:' in type: elif 'stypes:' in type:
vValue = val vValue = val
else: else:
vValue = val vValue = val
@ -275,7 +420,7 @@ class SonicYangExtMixin:
#create a dict to map each key under primary key with a dict yang model. #create a dict to map each key under primary key with a dict yang model.
#This is done to improve performance of mapping from values of TABLEs in #This is done to improve performance of mapping from values of TABLEs in
#config DB to leaf in YANG LIST. #config DB to leaf in YANG LIST.
leafDict = self._createLeafDict(model) leafDict = self._createLeafDict(model, table)
# get keys from YANG model list itself # get keys from YANG model list itself
listKeys = model['key']['@value'] listKeys = model['key']['@value']
@ -380,7 +525,7 @@ class SonicYangExtMixin:
self._xlateContainerInContainer(modelContainer, yang, configC, table) self._xlateContainerInContainer(modelContainer, yang, configC, table)
## Handle other leaves in container, ## Handle other leaves in container,
leafDict = self._createLeafDict(model) leafDict = self._createLeafDict(model, table)
vKeys = list(configC.keys()) vKeys = list(configC.keys())
for vKey in vKeys: for vKey in vKeys:
#vkey must be a leaf\leaf-list\choice in container #vkey must be a leaf\leaf-list\choice in container
@ -494,7 +639,7 @@ class SonicYangExtMixin:
# create a dict to map each key under primary key with a dict yang model. # create a dict to map each key under primary key with a dict yang model.
# This is done to improve performance of mapping from values of TABLEs in # This is done to improve performance of mapping from values of TABLEs in
# config DB to leaf in YANG LIST. # config DB to leaf in YANG LIST.
leafDict = self._createLeafDict(model) leafDict = self._createLeafDict(model, table)
# list with name <NAME>_LIST should be removed, # list with name <NAME>_LIST should be removed,
if "_LIST" in model['@name']: if "_LIST" in model['@name']:
@ -559,7 +704,7 @@ class SonicYangExtMixin:
self._revXlateContainerInContainer(modelContainer, yang, config, table) self._revXlateContainerInContainer(modelContainer, yang, config, table)
## Handle other leaves in container, ## Handle other leaves in container,
leafDict = self._createLeafDict(model) leafDict = self._createLeafDict(model, table)
for vKey in yang: for vKey in yang:
#vkey must be a leaf\leaf-list\choice in container #vkey must be a leaf\leaf-list\choice in container
if leafDict.get(vKey): if leafDict.get(vKey):

View File

@ -341,10 +341,8 @@ class Test_SonicYang(object):
else: else:
print("Xlate and Rev Xlate failed") print("Xlate and Rev Xlate failed")
# print for better debugging, in case of failure. # print for better debugging, in case of failure.
print("syc.jIn: {}".format({t:syc.jIn[t].keys() \ from jsondiff import diff
for t in syc.jIn.keys()})) print(diff(syc.jIn, syc.revXlateJson, syntax='symmetric'))
print("syc.revXlateJson: {}".format({t:syc.revXlateJson[t].keys() \
for t in syc.revXlateJson.keys()}))
# make it fail # make it fail
assert False == True assert False == True

View File

@ -928,7 +928,7 @@
}, },
"AAA": { "AAA": {
"authentication": { "authentication": {
"login": "local" "login": "local"
} }
}, },
"TACPLUS": { "TACPLUS": {
@ -1009,6 +1009,10 @@
"rrclient":"0" "rrclient":"0"
}, },
"default|192.168.1.1": { "default|192.168.1.1": {
"local_asn": "65200",
"asn": "65100",
"name": "bgp peer 65100",
"ebgp_multihop_ttl": "3"
} }
}, },
"BGP_NEIGHBOR_AF": { "BGP_NEIGHBOR_AF": {
@ -1043,7 +1047,7 @@
}, },
"PREFIX_SET": { "PREFIX_SET": {
"prefix1": { "prefix1": {
} }
}, },
"PREFIX": { "PREFIX": {
"prefix1|1|10.0.0.0/8|8..16": { "prefix1|1|10.0.0.0/8|8..16": {
@ -1093,5 +1097,5 @@
"Error": "This Table is for testing, This Table does not have YANG models." "Error": "This Table is for testing, This Table does not have YANG models."
} }
} }
} }