Migrate DEVICE_METADATA to db (#919)

dockers/docker-fpm-frr/bgpd.conf.j2

@@ -7,7 +7,7 @@
 {% endblock banner %}
 !
 {% block system_init %}
-hostname {{ inventory_hostname }}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
 password zebra
 log syslog informational
 log facility local4
@@ -51,7 +51,7 @@ router bgp {{ DEVICE_METADATA['localhost']['bgp_asn'] }}
 {% if bgp_session['asn'] != 0 %}
   neighbor {{ neighbor_addr }} remote-as {{ bgp_session['asn'] }}
   neighbor {{ neighbor_addr }} description {{ bgp_session['name'] }}
-{% if minigraph_devices[inventory_hostname]['type'] == 'ToRRouter' %}
+{% if DEVICE_METADATA['localhost']['type'] == 'ToRRouter' %}
   neighbor {{ neighbor_addr }} allowas-in 1
 {% endif %}
 {% if neighbor_addr | ipv4 %}

dockers/docker-fpm-frr/zebra.conf.j2

@@ -7,7 +7,7 @@
 {% endblock banner %}
 !
 {% block sys_init %}
-hostname {{ inventory_hostname }}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
 password zebra
 enable password zebra
 {% endblock sys_init %}

dockers/docker-fpm-gobgp/zebra.conf.j2

@@ -7,7 +7,7 @@
 {% endblock banner %}
 !
 {% block sys_init %}
-hostname {{ inventory_hostname }}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
 password zebra
 enable password zebra
 {% endblock sys_init %}

dockers/docker-fpm-quagga/bgpd.conf.j2

@@ -7,7 +7,7 @@
 {% endblock banner %}
 !
 {% block system_init %}
-hostname {{ inventory_hostname }}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
 password zebra
 log syslog informational
 log facility local4
@@ -27,7 +27,7 @@ router bgp {{ DEVICE_METADATA['localhost']['bgp_asn'] }}
   bgp log-neighbor-changes
   bgp bestpath as-path multipath-relax
 {# Advertise graceful restart capability for ToR #}
-{% if minigraph_devices[inventory_hostname]['type'] == 'ToRRouter' %}
+{% if DEVICE_METADATA['localhost']['type'] == 'ToRRouter' %}
   bgp graceful-restart
 {% endif %}
 {# TODO: use lo[0] for backward compatibility, will revisit the case with multiple lo interfaces #}
@@ -64,13 +64,13 @@ router bgp {{ DEVICE_METADATA['localhost']['bgp_asn'] }}
   neighbor {{ neighbor_addr }} shutdown
 {% endif %}
 {% if neighbor_addr | ipv4 %}
-{% if minigraph_devices[inventory_hostname]['type'] == 'ToRRouter' %}
+{% if DEVICE_METADATA['localhost']['type'] == 'ToRRouter' %}
   neighbor {{ neighbor_addr }} allowas-in 1
 {% endif %}
 {% endif %}
 {% if neighbor_addr | ipv6 %}
   address-family ipv6
-{% if minigraph_devices[inventory_hostname]['type'] == 'ToRRouter' %}
+{% if DEVICE_METADATA['locahost']['type'] == 'ToRRouter' %}
     neighbor {{ neighbor_addr }} allowas-in 1
 {% endif %}
     neighbor {{ neighbor_addr }} activate

dockers/docker-fpm-quagga/zebra.conf.j2

@@ -7,7 +7,7 @@
 {% endblock banner %}
 !
 {% block sys_init %}
-hostname {{ inventory_hostname }}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
 password zebra
 enable password zebra
 {% endblock sys_init %}

src/sonic-config-engine/minigraph.py

@@ -458,14 +458,15 @@ def parse_xml(filename, platform=None, port_config_file=None):
             ethernet_interfaces = parse_deviceinfo(child, hwsku)
 
     results = {}
-    results['minigraph_hwsku'] = hwsku
-    # sorting by lambdas are not easily done without custom filters.
-    # TODO: add jinja2 filter to accept a lambda to sort a list of dictionaries by attribute.
-    # TODO: alternatively (preferred), implement class containers for multiple-attribute entries, enabling sort by attr
     results['BGP_NEIGHBOR'] = bgp_sessions
-    results['DEVICE_METADATA'] = {'localhost': { 'bgp_asn': bgp_asn }}
+    results['DEVICE_METADATA'] = {'localhost': { 
+        'bgp_asn': bgp_asn,
+        'deployment_id': deployment_id,
+        'hostname': hostname,
+        'hwsku': hwsku,
+        'type': devices[hostname]['type']
+        }}
     results['BGP_PEER_RANGE'] = bgp_peers_with_range
-    # TODO: sort does not work properly on all interfaces of varying lengths. Need to sort by integer group(s).
 
     phyport_intfs = []
     vlan_intfs = []
@@ -496,14 +497,11 @@ def parse_xml(filename, platform=None, port_config_file=None):
     if devices != None:
         results['minigraph_console'] = get_console_info(devices, console_dev, console_port)
         results['minigraph_mgmt'] = get_mgmt_info(devices, mgmt_dev, mgmt_port)
-    results['minigraph_hostname'] = hostname
-    results['inventory_hostname'] = hostname
     results['syslog_servers'] = syslog_servers
     results['dhcp_servers'] = dhcp_servers
     results['ntp_servers'] = ntp_servers
     results['forced_mgmt_routes'] = mgmt_routes
     results['erspan_dst'] = erspan_dst
-    results['deployment_id'] = deployment_id
     results['ethernet_interfaces'] = ethernet_interfaces
 
     return results
@@ -513,9 +511,10 @@ def parse_device_desc_xml(filename):
     (lo_prefix, mgmt_prefix, hostname, hwsku, d_type) = parse_device(root)
 
     results = {}
-    results['minigraph_hwsku'] = hwsku
+    results['DEVICE_METADATA'] = {'localhost': { 
-    results['minigraph_hostname'] = hostname
+        'hostname': hostname,
-    results['inventory_hostname'] = hostname
+        'hwsku': hwsku,
+        }}
 
     lo_intfs = []
     ipn = ipaddress.IPNetwork(lo_prefix)

src/sonic-config-engine/sonic-cfggen

@@ -22,6 +22,7 @@ import yaml
 import jinja2
 import netaddr
 import json
+from functools import partial
 from minigraph import minigraph_encoder
 from minigraph import parse_xml
 from minigraph import parse_device_desc_xml
@@ -53,6 +54,16 @@ def is_ipv6(value):
             return False
     return addr.version == 6
 
+def prefix_attr(attr, value):
+    if not value:
+        return None
+    else:
+        try:
+            prefix = netaddr.IPNetwork(str(value))
+        except:
+            return None
+    return str(getattr(prefix, attr))
+
 def unique_name(l):
     name_list = []
     new_list = []
@@ -158,6 +169,8 @@ def main():
         env.filters['ipv4'] = is_ipv4
         env.filters['ipv6'] = is_ipv6
         env.filters['unique_name'] = unique_name
+        for attr in ['ip', 'network', 'prefixlen', 'netmask']:
+            env.filters[attr] = partial(prefix_attr, attr)
         template = env.get_template(template_file)
         print template.render(data)
     

src/sonic-config-engine/tests/test_cfggen.py

@@ -31,7 +31,7 @@ class TestCfgGen(TestCase):
         self.assertEqual(output, '')
     
     def test_device_desc(self):
-        argument = '-v minigraph_hwsku -M "' + self.sample_device_desc + '"'
+        argument = '-v "DEVICE_METADATA[\'localhost\'][\'hwsku\']" -M "' + self.sample_device_desc + '"'
         output = self.run_script(argument)
         self.assertEqual(output.strip(), 'ACS-MSN2700')
 
@@ -41,7 +41,7 @@ class TestCfgGen(TestCase):
         self.assertEqual(output.strip(), '10.0.1.5')
 
     def test_minigraph_sku(self):
-        argument = '-v minigraph_hwsku -m "' + self.sample_graph + '"'
+        argument = '-v "DEVICE_METADATA[\'localhost\'][\'hwsku\']" -m "' + self.sample_graph + '"'
         output = self.run_script(argument)
         self.assertEqual(output.strip(), 'Force10-Z9100')
     
@@ -51,7 +51,7 @@ class TestCfgGen(TestCase):
         self.assertTrue(len(output.strip()) > 0)
     
     def test_jinja_expression(self):
-        argument = '-m "' + self.sample_graph + '" -v "minigraph_devices[minigraph_hostname][\'type\']"'
+        argument = '-m "' + self.sample_graph + '" -v "DEVICE_METADATA[\'localhost\'][\'type\']"'
         output = self.run_script(argument)
         self.assertEqual(output.strip(), 'LeafRouter')
     
@@ -116,7 +116,7 @@ class TestCfgGen(TestCase):
         self.assertEqual(output.strip(), "[{'name': 'BGPSLBPassive', 'ip_range': ['10.10.10.10/26', '100.100.100.100/26']}]")
 
     def test_minigraph_deployment_id(self):
-        argument = '-m "' + self.sample_graph_bgp_speaker + '" -p "' + self.port_config + '" -v deployment_id'
+        argument = '-m "' + self.sample_graph_bgp_speaker + '" -p "' + self.port_config + '" -v "DEVICE_METADATA[\'localhost\'][\'deployment_id\']"'
         output = self.run_script(argument)
         self.assertEqual(output.strip(), "1")