Minigraph ECMP parsing support (cleaner format) (#4985)

Why I did it
To support FG_ECMP  scenarios
- How I did it
Modified minigraph parser to parse ECMP fields in the case they are present in minigraph
- How to verify it
Loaded ensuing config_db file on a DUT to verify the fields are parsed and configure device correctly
This commit is contained in:
kktheballer 2020-12-30 15:18:21 -08:00 committed by GitHub
parent a1fe203788
commit ba92a081ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 1838 additions and 8 deletions

View File

@ -10,6 +10,7 @@ from collections import defaultdict
from lxml import etree as ET
from lxml.etree import QName
from portconfig import get_port_config
from sonic_py_common.multi_asic import get_asic_id_from_name
from sonic_py_common.interface import backplane_prefix
@ -105,7 +106,73 @@ def parse_device(device):
return (lo_prefix, lo_prefix_v6, mgmt_prefix, name, hwsku, d_type, deployment_id)
def parse_png(png, hname):
def calculate_lcm_for_ecmp (nhdevices_bank_map, nhip_bank_map):
banks_enumerated = {}
lcm_array = []
for value in nhdevices_bank_map.values():
for key in nhip_bank_map.keys():
if nhip_bank_map[key] == value:
if value not in banks_enumerated:
banks_enumerated[value] = 1
else:
banks_enumerated[value] = banks_enumerated[value] + 1
for bank_enumeration in banks_enumerated.values():
lcm_list = range(1, bank_enumeration+1)
lcm_comp = lcm_list[0]
for i in lcm_list[1:]:
lcm_comp = lcm_comp * i / calculate_gcd(lcm_comp, i)
lcm_array.append(lcm_comp)
LCM = sum(lcm_array)
return int(LCM)
def calculate_gcd(x, y):
while y != 0:
(x, y) = (y, x % y)
return int(x)
def formulate_fine_grained_ecmp(version, dpg_ecmp_content, port_device_map, port_alias_map):
family = ""
tag = ""
neigh_key = []
if version == "ipv4":
family = "IPV4"
tag = "fgnhg_v4"
elif version == "ipv6":
family = "IPV6"
tag = "fgnhg_v6"
port_nhip_map = dpg_ecmp_content['port_nhip_map']
nhgaddr = dpg_ecmp_content['nhgaddr']
nhg_int = dpg_ecmp_content['nhg_int']
nhip_device_map = {port_nhip_map[x]: port_device_map[x] for x in port_device_map
if x in port_nhip_map}
nhip_devices = sorted(list(set(nhip_device_map.values())))
nhdevices_ip_bank_map = {device: bank for bank, device in enumerate(nhip_devices)}
nhip_bank_map = {ip: nhdevices_ip_bank_map[device] for ip, device in nhip_device_map.items()}
LCM = calculate_lcm_for_ecmp(nhdevices_ip_bank_map, nhip_bank_map)
FG_NHG_MEMBER = {ip: {"FG_NHG": tag, "bank": bank} for ip, bank in nhip_bank_map.items()}
nhip_port_map = dict(zip(port_nhip_map.values(), port_nhip_map.keys()))
for nhip, memberinfo in FG_NHG_MEMBER.items():
if nhip in nhip_port_map:
memberinfo["link"] = port_alias_map[nhip_port_map[nhip]]
FG_NHG_MEMBER[nhip] = memberinfo
FG_NHG_PREFIX = {nhgaddr: {"FG_NHG": tag}}
FG_NHG = {tag: {"bucket_size": LCM}}
for ip in nhip_bank_map:
neigh_key.append(str(nhg_int + "|" + ip))
NEIGH = {neigh_key: {"family": family} for neigh_key in neigh_key}
fine_grained_content = {"FG_NHG_MEMBER": FG_NHG_MEMBER, "FG_NHG": FG_NHG, "FG_NHG_PREFIX": FG_NHG_PREFIX, "NEIGH": NEIGH}
return fine_grained_content
def parse_png(png, hname, dpg_ecmp_content = None):
neighbors = {}
devices = {}
console_dev = ''
@ -116,6 +183,13 @@ def parse_png(png, hname):
console_ports = {}
mux_cable_ports = {}
is_storage_device = False
port_device_map = {}
png_ecmp_content = {}
FG_NHG_MEMBER = {}
FG_NHG_PREFIX = {}
FG_NHG = {}
NEIGH = {}
for child in png:
if child.tag == str(QName(ns, "DeviceInterfaceLinks")):
for link in child.findall(str(QName(ns, "DeviceLinkBase"))):
@ -141,6 +215,11 @@ def parse_png(png, hname):
}
continue
if linktype == "DeviceInterfaceLink":
endport = link.find(str(QName(ns, "EndPort"))).text
startdevice = link.find(str(QName(ns, "StartDevice"))).text
port_device_map[endport] = startdevice
if linktype != "DeviceInterfaceLink" and linktype != "UnderlayInterfaceLink":
continue
@ -196,6 +275,7 @@ def parse_png(png, hname):
elif node.tag == str(QName(ns, "EndDevice")):
mgmt_dev = node.text
if child.tag == str(QName(ns, "DeviceInterfaceLinks")):
for link in child.findall(str(QName(ns, 'DeviceLinkBase'))):
if link.find(str(QName(ns, "ElementType"))).text == "LogicalLink":
@ -205,7 +285,19 @@ def parse_png(png, hname):
mux_cable_ports[intf_name] = "true"
return (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speeds, console_ports, mux_cable_ports, is_storage_device)
if (len(dpg_ecmp_content)):
for version, content in dpg_ecmp_content.items(): # version is ipv4 or ipv6
fine_grained_content = formulate_fine_grained_ecmp(version, content, port_device_map, port_alias_map) # port_alias_map
FG_NHG_MEMBER.update(fine_grained_content['FG_NHG_MEMBER'])
FG_NHG_PREFIX.update(fine_grained_content['FG_NHG_PREFIX'])
FG_NHG.update(fine_grained_content['FG_NHG'])
NEIGH.update(fine_grained_content['NEIGH'])
png_ecmp_content = {"FG_NHG_PREFIX": FG_NHG_PREFIX, "FG_NHG_MEMBER": FG_NHG_MEMBER, "FG_NHG": FG_NHG,
"NEIGH": NEIGH}
return (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speeds, console_ports, mux_cable_ports, is_storage_device, png_ecmp_content)
def parse_asic_external_link(link, asic_name, hostname):
neighbors = {}
@ -295,6 +387,8 @@ def parse_asic_png(png, asic_name, hostname):
if lo_prefix_v6:
device_data['lo_addr_v6']= lo_prefix_v6
devices[name] = device_data
return (neighbors, devices, port_speeds)
def parse_loopback_intf(child):
@ -339,12 +433,13 @@ def parse_dpg(dpg, hname):
ipintfs = child.find(str(QName(ns, "IPInterfaces")))
intfs = {}
ip_intfs_map = {}
for ipintf in ipintfs.findall(str(QName(ns, "IPInterface"))):
intfalias = ipintf.find(str(QName(ns, "AttachTo"))).text
intfname = port_alias_map.get(intfalias, intfalias)
ipprefix = ipintf.find(str(QName(ns, "Prefix"))).text
intfs[(intfname, ipprefix)] = {}
ip_intfs_map[ipprefix] = intfalias
lo_intfs = parse_loopback_intf(child)
mvrfConfigs = child.find(str(QName(ns, "MgmtVrfConfigs")))
@ -381,7 +476,46 @@ def parse_dpg(dpg, hname):
pcs[pcintfname] = {'members': pcmbr_list, 'fallback': pcintf.find(str(QName(ns, "Fallback"))).text, 'min_links': str(int(math.ceil(len() * 0.75)))}
else:
pcs[pcintfname] = {'members': pcmbr_list, 'min_links': str(int(math.ceil(len(pcmbr_list) * 0.75)))}
port_nhipv4_map = {}
port_nhipv6_map = {}
nhgaddr = ["", ""]
nhg_int = ""
nhportlist = []
dpg_ecmp_content = {}
ipnhs = child.find(str(QName(ns, "IPNextHops")))
if ipnhs is not None:
for ipnh in ipnhs.findall(str(QName(ns, "IPNextHop"))):
if ipnh.find(str(QName(ns, "Type"))).text == 'FineGrainedECMPGroupMember':
ipnhfmbr = ipnh.find(str(QName(ns, "AttachTo"))).text
ipnhaddr = ipnh.find(str(QName(ns, "Address"))).text
nhportlist.append(ipnhfmbr)
if "." in ipnhaddr:
port_nhipv4_map[ipnhfmbr] = ipnhaddr
elif ":" in ipnhaddr:
port_nhipv6_map[ipnhfmbr] = ipnhaddr
if port_nhipv4_map is not None and port_nhipv6_map is not None:
subnet_check_ip = list(port_nhipv4_map.values())[0]
for subnet_range in ip_intfs_map:
if ("." in subnet_range):
a = ipaddress.ip_address(UNICODE_TYPE(subnet_check_ip))
n = list(ipaddress.ip_network(UNICODE_TYPE(subnet_range), False).hosts())
if a in n:
nhg_int = ip_intfs_map[subnet_range]
dwnstrms = child.find(str(QName(ns, "DownstreamSummarySet")))
for dwnstrm in dwnstrms.findall(str(QName(ns, "DownstreamSummary"))):
dwnstrmentry = str(ET.tostring(dwnstrm))
if ("FineGrainedECMPGroupDestination" in dwnstrmentry):
subnet_ip = dwnstrm.find(str(QName(ns1, "Subnet"))).text
truncsubnet_ip = subnet_ip.split("/")[0]
if "." in (truncsubnet_ip):
nhgaddr[0] = subnet_ip
elif ":" in (truncsubnet_ip):
nhgaddr[1] = subnet_ip
ipv4_content = {"port_nhip_map": port_nhipv4_map, "nhgaddr": nhgaddr[0], "nhg_int": nhg_int}
ipv6_content = {"port_nhip_map": port_nhipv6_map, "nhgaddr": nhgaddr[1], "nhg_int": nhg_int}
dpg_ecmp_content['ipv4'] = ipv4_content
dpg_ecmp_content['ipv6'] = ipv6_content
vlanintfs = child.find(str(QName(ns, "VlanInterfaces")))
vlan_intfs = []
vlans = {}
@ -503,6 +637,7 @@ def parse_dpg(dpg, hname):
except:
print("Warning: Ignoring Control Plane ACL %s without type" % aclname, file=sys.stderr)
mg_tunnels = child.find(str(QName(ns, "TunnelInterfaces")))
if mg_tunnels is not None:
table_key_to_mg_key_map = {"encap_ecn_mode": "EcnEncapsulationMode",
@ -521,7 +656,7 @@ def parse_dpg(dpg, hname):
if mg_key in mg_tunnel.attrib:
tunnelintfs[tunnel_type][tunnel_name][table_key] = mg_tunnel.attrib[mg_key]
return intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnelintfs
return intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnelintfs, dpg_ecmp_content
return None, None, None, None, None, None, None, None, None, None
def parse_host_loopback(dpg, hname):
@ -927,6 +1062,8 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
bgp_monitors = []
bgp_asn = None
intfs = None
dpg_ecmp_content = {}
png_ecmp_content = {}
vlan_intfs = None
pc_intfs = None
tunnel_intfs = None
@ -988,13 +1125,13 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
for child in root:
if asic_name is None:
if child.tag == str(QName(ns, "DpgDec")):
(intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs) = parse_dpg(child, hostname)
(intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs, dpg_ecmp_content) = parse_dpg(child, hostname)
elif child.tag == str(QName(ns, "CpgDec")):
(bgp_sessions, bgp_internal_sessions, bgp_asn, bgp_peers_with_range, bgp_monitors) = parse_cpg(child, hostname)
elif child.tag == str(QName(ns, "PngDec")):
(neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speed_png, console_ports, mux_cable_ports, is_storage_device) = parse_png(child, hostname)
(neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speed_png, console_ports, mux_cable_ports, is_storage_device, png_ecmp_content) = parse_png(child, hostname, dpg_ecmp_content)
elif child.tag == str(QName(ns, "UngDec")):
(u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname)
(u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname, None)
elif child.tag == str(QName(ns, "MetadataDeclaration")):
(syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type) = parse_meta(child, hostname)
elif child.tag == str(QName(ns, "LinkMetadataDeclaration")):
@ -1003,7 +1140,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
(port_speeds_default, port_descriptions) = parse_deviceinfo(child, hwsku)
else:
if child.tag == str(QName(ns, "DpgDec")):
(intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs) = parse_dpg(child, asic_name)
(intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs, dpg_ecmp_content) = parse_dpg(child, asic_name)
host_lo_intfs = parse_host_loopback(child, hostname)
elif child.tag == str(QName(ns, "CpgDec")):
(bgp_sessions, bgp_internal_sessions, bgp_asn, bgp_peers_with_range, bgp_monitors) = parse_cpg(child, asic_name, local_devices)
@ -1305,6 +1442,13 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
'client_crt_cname': 'client.restapi.sonic'
}
}
if len(png_ecmp_content):
results['FG_NHG_MEMBER'] = png_ecmp_content['FG_NHG_MEMBER']
results['FG_NHG_PREFIX'] = png_ecmp_content['FG_NHG_PREFIX']
results['FG_NHG'] = png_ecmp_content['FG_NHG']
results['NEIGH'] = png_ecmp_content['NEIGH']
# Do not configure the minigraph's mirror session, which is currently unused
# mirror_sessions = {}
# if erspan_dst:

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,65 @@
# name lanes alias index
Ethernet0 0,1,2,3 etp1 1
Ethernet4 4,5,6,7 etp2 2
Ethernet8 8,9,10,11 etp3 3
Ethernet12 12,13,14,15 etp4 4
Ethernet16 16,17,18,19 etp5 5
Ethernet20 20,21,22,23 etp6 6
Ethernet24 24,25,26,27 etp7 7
Ethernet28 28,29,30,31 etp8 8
Ethernet32 32,33,34,35 etp9 9
Ethernet36 36,37,38,39 etp10 10
Ethernet40 40,41,42,43 etp11 11
Ethernet44 44,45,46,47 etp12 12
Ethernet48 48,49,50,51 etp13 13
Ethernet52 52,53,54,55 etp14 14
Ethernet56 56,57,58,59 etp15 15
Ethernet60 60,61,62,63 etp16 16
Ethernet64 64,65,66,67 etp17 17
Ethernet68 68,69,70,71 etp18 18
Ethernet72 72,73,74,75 etp19 19
Ethernet76 76,77,78,79 etp20 20
Ethernet80 80,81,82,83 etp21 21
Ethernet84 84,85,86,87 etp22 22
Ethernet88 88,89,90,91 etp23 23
Ethernet92 92,93,94,95 etp24 24
Ethernet96 96,97,98,99 etp25 25
Ethernet100 100,101,102,103 etp26 26
Ethernet104 104,105,106,107 etp27 27
Ethernet108 108,109,110,111 etp28 28
Ethernet112 112,113,114,115 etp29 29
Ethernet116 116,117,118,119 etp30 30
Ethernet120 120,121,122,123 etp31 31
Ethernet124 124,125,126,127 etp32 32
Ethernet128 128,129,130,131 etp33 33
Ethernet132 132,133,134,135 etp34 34
Ethernet136 136,137,138,139 etp35 35
Ethernet140 140,141,142,143 etp36 36
Ethernet144 144,145,146,147 etp37 37
Ethernet148 148,149,150,151 etp38 38
Ethernet152 152,153,154,155 etp39 39
Ethernet156 156,157,158,159 etp40 40
Ethernet160 160,161,162,163 etp41 41
Ethernet164 164,165,166,167 etp42 42
Ethernet168 168,169,170,171 etp43 43
Ethernet172 172,173,174,175 etp44 44
Ethernet176 176,177,178,179 etp45 45
Ethernet180 180,181,182,183 etp46 46
Ethernet184 184,185,186,187 etp47 47
Ethernet188 188,189,190,191 etp48 48
Ethernet192 192,193,194,195 etp49 49
Ethernet196 196,197,198,199 etp50 50
Ethernet200 200,201,202,203 etp51 51
Ethernet204 204,205,206,207 etp52 52
Ethernet208 208,209,210,211 etp53 53
Ethernet212 212,213,214,215 etp54 54
Ethernet216 216,217,218,219 etp55 55
Ethernet220 220,221,222,223 etp56 56
Ethernet224 224,225,226,227 etp57 57
Ethernet228 228,229,230,231 etp58 58
Ethernet232 232,233,234,235 etp59 59
Ethernet236 236,237,238,239 etp60 60
Ethernet240 240,241,242,243 etp61 61
Ethernet244 244,245,246,247 etp62 62
Ethernet248 248,249,250,251 etp63 63
Ethernet252 252,253,254,255 etp64 64

View File

@ -23,8 +23,10 @@ class TestCfgGen(TestCase):
self.sample_graph_bgp_speaker = os.path.join(self.test_dir, 't0-sample-bgp-speaker.xml')
self.sample_device_desc = os.path.join(self.test_dir, 'device.xml')
self.port_config = os.path.join(self.test_dir, 't0-sample-port-config.ini')
self.mlnx_port_config = os.path.join(self.test_dir, 'mellanox-sample-port-config.ini')
self.output_file = os.path.join(self.test_dir, 'output')
self.output2_file = os.path.join(self.test_dir, 'output2')
self.ecmp_graph = os.path.join(self.test_dir, 'fg-ecmp-sample-minigraph.xml')
def tearDown(self):
try:
@ -222,6 +224,34 @@ class TestCfgGen(TestCase):
output = self.run_script(argument)
self.assertEqual(output.strip(), "[('Vlan1000', '192.168.0.1/27'), 'Vlan1000']")
def test_minigraph_ecmp_fg_nhg(self):
argument = '-m "' + self.ecmp_graph + '" -p "' + self.mlnx_port_config + '" -v \"FG_NHG.values()|list\"'
output = self.run_script(argument)
self.assertEqual(output.strip(), "[{'bucket_size': 120}, {'bucket_size': 120}]")
def test_minigraph_ecmp_members(self):
argument = '-m "' + self.ecmp_graph + '" -p "' + self.mlnx_port_config + '" -v "FG_NHG_MEMBER.keys()|list|sort"'
output = self.run_script(argument)
self.assertEqual(output.strip(), "['200.200.200.1', '200.200.200.10', '200.200.200.2', '200.200.200.3', '200.200.200.4', '200.200.200.5',"
" '200.200.200.6', '200.200.200.7', '200.200.200.8', '200.200.200.9', '200:200:200:200::1', '200:200:200:200::10',"
" '200:200:200:200::2', '200:200:200:200::3', '200:200:200:200::4', '200:200:200:200::5', '200:200:200:200::6',"
" '200:200:200:200::7', '200:200:200:200::8', '200:200:200:200::9']")
def test_minigraph_ecmp_neighbors(self):
argument = '-m "' + self.ecmp_graph + '" -p "' + self.mlnx_port_config + '" -v "NEIGH.keys()|list|sort"'
output = self.run_script(argument)
self.assertEqual(output.strip(), "['Vlan31|200.200.200.1', 'Vlan31|200.200.200.10', 'Vlan31|200.200.200.2', 'Vlan31|200.200.200.3',"
" 'Vlan31|200.200.200.4', 'Vlan31|200.200.200.5', 'Vlan31|200.200.200.6', 'Vlan31|200.200.200.7',"
" 'Vlan31|200.200.200.8', 'Vlan31|200.200.200.9', 'Vlan31|200:200:200:200::1', 'Vlan31|200:200:200:200::10',"
" 'Vlan31|200:200:200:200::2', 'Vlan31|200:200:200:200::3', 'Vlan31|200:200:200:200::4', 'Vlan31|200:200:200:200::5', "
"'Vlan31|200:200:200:200::6', 'Vlan31|200:200:200:200::7', 'Vlan31|200:200:200:200::8', 'Vlan31|200:200:200:200::9']")
def test_minigraph_ecmp_prefixes(self):
argument = '-m "' + self.ecmp_graph + '" -p "' + self.mlnx_port_config + '" -v "FG_NHG_PREFIX.keys()|list|sort"'
output = self.run_script(argument)
self.assertEqual(output.strip(), "['100.50.25.12/32', 'fc:5::/128']")
def test_minigraph_portchannels(self):
argument = '-m "' + self.sample_graph_simple + '" -p "' + self.port_config + '" -v PORTCHANNEL'
output = self.run_script(argument)