[YANG] Fix issue: Non compliant leaf list in config_db schema (#10291) (#10768)

Fix issue: Non compliant leaf list in config_db schema: https://github.com/Azure/sonic-buildimage/issues/9801

The basic flow of DPB is like:
1.	Transfer config db json value to YANG json value, name it “yangIn”
2.	Validate “yangIn” by libyang
3.	Generate a YANG json value to represent the target configuration, name it “yangTarget”
4.	Do diff between “yangIn” and “yangTarget”
5.	Apply the diff to CONFIG DB json and save it back to DB

The fix:
•	For step #1, If value of a leaf-list field string type, transfer it to a list by splitting it with “,” the purpose here is to make step#2 happy. We also need to save <table_name>.<key>.<field_name> to a set named “leaf_list_with_string_value_set”.
•	For step#5, loop “leaf_list_with_string_value_set” and change those fields back to a string.

1. Manual test
2. Changed sample config DB and unit test passed

Conflicts:
	src/sonic-yang-mgmt/sonic_yang_ext.py

src/sonic-yang-mgmt/sonic_yang.py

@@ -44,7 +44,9 @@ class SonicYang(SonicYangExtMixin):
         # below dict will store preProcessed yang objects, which may be needed by
         # all yang modules, such as grouping.
         self.preProcessedYang = dict()
-
+        # element path for CONFIG DB. An example for this list could be:
+        # ['PORT', 'Ethernet0', 'speed']
+        self.elementPath = []
         try:
             self.ctx = ly.Context(yang_dir, sonic_yang_options)
         except Exception as e:

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -15,6 +15,19 @@ qos_maps_model = ['DSCP_TO_TC_MAP_LIST',
             'MAP_PFC_PRIORITY_TO_QUEUE_LIST',
             'PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_LIST']
 
+# Workaround for those fields who is defined as leaf-list in YANG model but have string value in config DB.
+# Dictinary structure key = (<table_name>, <field_name>), value = seperator
+LEAF_LIST_WITH_STRING_VALUE_DICT = {
+    ('MIRROR_SESSION', 'src_ip'): ',',
+    ('NTP', 'src_intf'): ';',
+    ('BGP_ALLOWED_PREFIXES', 'prefixes_v4'): ',',
+    ('BGP_ALLOWED_PREFIXES', 'prefixes_v6'): ',',
+    ('BUFFER_PORT_EGRESS_PROFILE_LIST', 'profile_list'): ',',
+    ('BUFFER_PORT_INGRESS_PROFILE_LIST', 'profile_list'): ',',
+    ('PORT', 'adv_speeds'): ',',
+    ('PORT', 'adv_interface_types'): ',',
+}
+
 """
 This is the Exception thrown out of all public function of this class.
 """
@@ -402,6 +415,11 @@ class SonicYangExtMixin:
         # if it is a leaf-list do it for each element
         if leafDict[key]['__isleafList']:
             vValue = list()
+            if isinstance(value, str) and (self.elementPath[0], self.elementPath[-1]) in LEAF_LIST_WITH_STRING_VALUE_DICT:
+                # For field defined as leaf-list but has string value in CONFIG DB, need do special handling here. For exampe:
+                # port.adv_speeds in CONFIG DB has value "100,1000,10000", it shall be transferred to [100,1000,10000] as YANG value here to 
+                # make it align with its YANG definition.
+                value = (x.strip() for x in value.split(LEAF_LIST_WITH_STRING_VALUE_DICT[(self.elementPath[0], self.elementPath[-1])]))
             for v in value:
                 vValue.append(_yangConvert(v))
         else:
@@ -540,6 +558,7 @@ class SonicYangExtMixin:
         primaryKeys = list(config.keys())
         for pkey in primaryKeys:
             try:
+                self.elementPath.append(pkey)
                 vKey = None
                 self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\
                     format(pkey))
@@ -547,9 +566,13 @@ class SonicYangExtMixin:
                 keyDict = self._extractKey(pkey, listKeys)
                 # fill rest of the values in keyDict
                 for vKey in config[pkey]:
+                    self.elementPath.append(vKey)
                     self.sysLog(syslog.LOG_DEBUG, "xlateList vkey {}".format(vKey))
+                    try:
                         keyDict[vKey] = self._findYangTypedValue(vKey, \
                                             config[pkey][vKey], leafDict)
+                    finally:
+                        self.elementPath.pop()
                 yang.append(keyDict)
                 # delete pkey from config, done to match one key with one list
                 del config[pkey]
@@ -561,6 +584,8 @@ class SonicYangExtMixin:
                 exceptionList.append(str(e))
                 # with multilist, we continue matching other keys.
                 continue
+            finally:
+                self.elementPath.pop()
 
         return
 
@@ -591,8 +616,10 @@ class SonicYangExtMixin:
         if not configC.get(ccontainer['@name']):
             return
         self.sysLog(msg="xlateProcessListOfContainer: {}".format(ccontainer['@name']))
+        self.elementPath.append(ccontainer['@name'])
         self._xlateContainer(ccontainer, yang[ccontainer['@name']], \
         configC[ccontainer['@name']], table)
+        self.elementPath.pop()
         # clean empty container
         if len(yang[ccontainer['@name']]) == 0:
             del yang[ccontainer['@name']]
@@ -640,8 +667,10 @@ class SonicYangExtMixin:
         for vKey in vKeys:
             #vkey must be a leaf\leaf-list\choice in container
             if leafDict.get(vKey):
+                self.elementPath.append(vKey)
                 self.sysLog(syslog.LOG_DEBUG, "xlateContainer vkey {}".format(vKey))
                 yang[vKey] = self._findYangTypedValue(vKey, configC[vKey], leafDict)
+                self.elementPath.pop()
                 # delete entry from copy of config
                 del configC[vKey]
 
@@ -671,8 +700,10 @@ class SonicYangExtMixin:
             yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key]
             yangJ[key][subkey] = dict()
             self.sysLog(msg="xlateConfigDBtoYang {}:{}".format(key, subkey))
+            self.elementPath.append(table)
             self._xlateContainer(cmap['container'], yangJ[key][subkey], \
                                 jIn[table], table)
+            self.elementPath = []
 
         return
 
@@ -729,6 +760,11 @@ class SonicYangExtMixin:
 
         # if it is a leaf-list do it for each element
         if leafDict[key]['__isleafList']:
+            if isinstance(value, list) and (self.elementPath[0], self.elementPath[-1]) in LEAF_LIST_WITH_STRING_VALUE_DICT:
+                # For field defined as leaf-list but has string value in CONFIG DB, we need do special handling here:
+                # e.g. port.adv_speeds is [10,100,1000] in YANG, need to convert it into a string for CONFIG DB: "10,100,1000"
+                vValue = LEAF_LIST_WITH_STRING_VALUE_DICT[(self.elementPath[0], self.elementPath[-1])].join((_revYangConvert(x) for x in value))
+            else:
                 vValue = list()
                 for v in value:
                     vValue.append(_revYangConvert(v))
@@ -840,12 +876,16 @@ class SonicYangExtMixin:
                 # create key of config DB table
                 pkey, pkeydict = self._createKey(entry, listKeys)
                 self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey))
+                self.elementPath.append(pkey)
                 config[pkey]= dict()
                 # fill rest of the entries
                 for key in entry:
                     if key not in pkeydict:
+                        self.elementPath.append(key)
                         config[pkey][key] = self._revFindYangTypedValue(key, \
                             entry[key], leafDict)
+                        self.elementPath.pop()
+                self.elementPath.pop()
 
         return
 
@@ -869,8 +909,10 @@ class SonicYangExtMixin:
         if yang.get(modelContainer['@name']):
             config[modelContainer['@name']] = dict()
             self.sysLog(msg="revXlateContainerInContainer {}".format(modelContainer['@name']))
+            self.elementPath.append(modelContainer['@name'])
             self._revXlateContainer(modelContainer, yang[modelContainer['@name']], \
                 config[modelContainer['@name']], table)
+            self.elementPath.pop()
         return
 
     """
@@ -902,7 +944,9 @@ class SonicYangExtMixin:
             #vkey must be a leaf\leaf-list\choice in container
             if leafDict.get(vKey):
                 self.sysLog(syslog.LOG_DEBUG, "revXlateContainer vkey {}".format(vKey))
+                self.elementPath.append(vKey)
                 config[vKey] = self._revFindYangTypedValue(vKey, yang[vKey], leafDict)
+                self.elementPath.pop()
 
         return
 
@@ -930,8 +974,10 @@ class SonicYangExtMixin:
                 cDbJson[table] = dict()
                 #print(key + "--" + subkey)
                 self.sysLog(msg="revXlateYangtoConfigDB {}".format(table))
+                self.elementPath.append(table)
                 self._revXlateContainer(cmap['container'], yangJ[module_top][container], \
                     cDbJson[table], table)
+                self.elementPath = []
 
         return
 

src/sonic-yang-models/tests/files/sample_config_db.json

@@ -65,12 +65,12 @@
         },
         "BUFFER_PORT_INGRESS_PROFILE_LIST": {
             "Ethernet9": {
-                "profile_list": ["ingress_lossy_profile"]
+                "profile_list": "ingress_lossy_profile"
             }
         },
         "BUFFER_PORT_EGRESS_PROFILE_LIST": {
             "Ethernet9": {
-                "profile_list": ["egress_lossless_profile", "egress_lossy_profile"]
+                "profile_list": "egress_lossless_profile,egress_lossy_profile"
             }
         },
         "PORTCHANNEL": {
@@ -363,10 +363,7 @@
         "NTP": {
             "global": {
                 "vrf": "mgmt",
-                "src_intf": [
+                "src_intf": "eth0;Loopback0"
-                    "eth0",
-                    "Loopback0"
-                ]
             }
         },
         "NTP_SERVER": {
@@ -397,7 +394,10 @@
                 "description": "",
                 "speed": "11100",
                 "tpid": "0x8100",
-                "admin_status": "up"
+                "admin_status": "up",
+                "autoneg": "on",
+                "adv_speeds": "100000,50000",
+                "adv_interface_types": "CR,CR4"
             },
             "Ethernet2": {
                 "alias": "Eth1/3",
@@ -405,7 +405,10 @@
                 "description": "",
                 "speed": "11100",
                 "tpid": "0x8100",
-                "admin_status": "up"
+                "admin_status": "up",
+                "autoneg": "on",
+                "adv_speeds": "all",
+                "adv_interface_types": "all"
             },
             "Ethernet3": {
                 "alias": "Eth1/4",