sonic-buildimage/src/sonic-config-engine/sonic-cfggen

#!/usr/bin/env python
"""sonic-cfggen

A tool to read SONiC config data from one or more of the following sources:
minigraph file, config DB, json file(s), yaml files(s), command line input,
and write the data into DB, print as json, or render a jinja2 config template.

Examples:
    Render template with minigraph:
        sonic-cfggen -m -t /usr/share/template/bgpd.conf.j2
    Dump config DB content into json file:
        sonic-cfggen -d --print-data > db_dump.json
    Load content of json file into config DB:
        sonic-cfggen -j db_dump.json --write-to-db
See usage string for detail description for arguments.
"""

from __future__ import print_function

# monkey patch re.compile to do lazy regular expression compilation.
# This is done to improve import time of jinja2, yaml, natsort modules, because they
# do many regexp compilation at import time, so it will speed up sonic-cfggen invocations
# that do not require template generation or yaml loading. sonic-cfggen is used in so many places
# during system boot up that importing jinja2, yaml, natsort every time
# without lazy regular expression compilation affect boot up time.
# FIXME: remove this once sonic-cfggen and templates dependencies are replaced with a faster approach
import lazy_re

import sys
import os.path
import argparse
import yaml
import jinja2
import netaddr
import json
from functools import partial
from minigraph import minigraph_encoder
from minigraph import parse_xml
from minigraph import parse_device_desc_xml
from minigraph import parse_asic_sub_role
from portconfig import get_port_config
from sonic_device_util import get_machine_info
from sonic_device_util import get_platform_info
from sonic_device_util import get_system_mac
from sonic_device_util import get_npu_id_from_name
from config_samples import generate_sample_config
from config_samples import get_available_config
from swsssdk import SonicV2Connector, ConfigDBConnector, SonicDBConfig 
from redis_bcc import RedisBytecodeCache
from collections import OrderedDict
from natsort import natsorted

def sort_by_port_index(value):
    if not value:
        return
    if isinstance(value, list):
        value.sort(key = lambda k: int(k[8:]))

def is_ipv4(value):
    if not value:
        return False
    if isinstance(value, netaddr.IPNetwork):
        addr = value
    else:
        try:
            addr = netaddr.IPNetwork(str(value))
        except:
            return False
    return addr.version == 4

def is_ipv6(value):
    if not value:
        return False
    if isinstance(value, netaddr.IPNetwork):
        addr = value
    else:
        try:
            addr = netaddr.IPNetwork(str(value))
        except:
            return False
    return addr.version == 6

def prefix_attr(attr, value):
    if not value:
        return None
    else:
        try:
            prefix = netaddr.IPNetwork(str(value))
        except:
            return None
    return str(getattr(prefix, attr))

def unique_name(l):
    name_list = []
    new_list = []
    for item in l:
        if item['name'] not in name_list:
            name_list.append(item['name'])
            new_list.append(item)
    return new_list

def pfx_filter(value):
    """INTERFACE Table can have keys in one of the two formats:
       string or tuple - This filter skips the string keys and only
       take into account the tuple.
       For eg - VLAN_INTERFACE|Vlan1000 vs VLAN_INTERFACE|Vlan1000|192.168.0.1/21
    """
    table = OrderedDict()

    if not value:
        return table

    for key,val in value.items():
        if not isinstance(key, tuple):
            continue
        table[key] = val
    return table

def ip_network(value):
    """ Extract network for network prefix """
    try:
        r_v = netaddr.IPNetwork(value)
    except:
        return "Invalid ip address %s" % value
    return r_v.network

class FormatConverter:
    """Convert config DB based schema to legacy minigraph based schema for backward capability.
We will move to DB schema and remove this class when the config templates are modified.

TODO(taoyl): Current version of config db only supports BGP admin states.
    All other configuration are still loaded from minigraph. Plan to remove
    minigraph and move everything into config db in a later commit.
    """
    @staticmethod
    def db_to_output(db_data):
        return db_data

    @staticmethod
    def output_to_db(output_data):
        db_data = {}
        for table_name in output_data:
            if table_name[0].isupper():
                db_data[table_name] = output_data[table_name]
        return db_data

    @staticmethod
    def to_serialized(data, lookup_key = None):
        if type(data) is dict:
            data = OrderedDict(natsorted(data.items()))

            if lookup_key != None:
                newData = {}
                for key in data.keys():
                    if ((type(key) is unicode and lookup_key == key) or (type(key) is tuple and lookup_key in key)):
                        newData[ConfigDBConnector.serialize_key(key)] = data.pop(key)
                        break
                return newData

            for key in data.keys():
                new_key = ConfigDBConnector.serialize_key(key)
                if new_key != key:
                    data[new_key] = data.pop(key)
                data[new_key] = FormatConverter.to_serialized(data[new_key])
        return data

    @staticmethod
    def to_deserialized(data):
        for table in data:
            if type(data[table]) is dict:
                for key in data[table].keys():
                    new_key = ConfigDBConnector.deserialize_key(key)
                    if new_key != key:
                        data[table][new_key] = data[table].pop(key)
        return data


def deep_update(dst, src):
    for key, value in src.iteritems():
        if isinstance(value, dict):
             node = dst.setdefault(key, {})
             deep_update(node, value)
        else:
             dst[key] = value
    return dst

def sort_data(data):
    for table in data:
        if type(data[table]) is dict:
            data[table] = OrderedDict(natsorted(data[table].items()))
    return data


def main():
    parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml')
    group.add_argument("-M", "--device-description", help="device description xml file")
    group.add_argument("-k", "--hwsku", help="HwSKU")
    parser.add_argument("-n", "--namespace", help="namespace name", nargs='?', const=None, default=None)
    parser.add_argument("-p", "--port-config", help="port config file, used with -m or -k", nargs='?', const=None)
    parser.add_argument("-y", "--yaml", help="yaml file that contains additional variables", action='append', default=[])
    parser.add_argument("-j", "--json", help="json file that contains additional variables", action='append', default=[])
    parser.add_argument("-a", "--additional-data", help="addition data, in json string")
    parser.add_argument("-d", "--from-db", help="read config from configdb", action='store_true')
    parser.add_argument("-H", "--platform-info", help="read platform and hardware info", action='store_true')
    parser.add_argument("-s", "--redis-unix-sock-file", help="unix sock file for redis connection")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-t", "--template", help="render the data with the template file")
    parser.add_argument("-T", "--template_dir", help="search base for the template files", action='store')
    group.add_argument("-v", "--var", help="print the value of a variable, support jinja2 expression")
    group.add_argument("--var-json", help="print the value of a variable, in json format")
    group.add_argument("-w", "--write-to-db", help="write config into configdb", action='store_true')
    group.add_argument("--print-data", help="print all data", action='store_true')
    group.add_argument("--preset", help="generate sample configuration from a preset template", choices=get_available_config())
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-K", "--key", help="Lookup for a specific key")
    args = parser.parse_args()

    platform = get_platform_info(get_machine_info())

    db_kwargs = {}
    if args.redis_unix_sock_file != None:
        db_kwargs['unix_socket_path'] = args.redis_unix_sock_file

    data = {}
    hwsku = args.hwsku
    asic_name = args.namespace
    asic_id = None
    if asic_name is not None:
        asic_id = get_npu_id_from_name(asic_name)


    # Load the database config for the namespace from global database json
    if args.namespace is not None:
        SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace)

    if hwsku is not None:
        hardware_data = {'DEVICE_METADATA': {'localhost': {
            'hwsku': hwsku
            }}}
        deep_update(data, hardware_data)
        (ports, _, _) = get_port_config(hwsku, platform, args.port_config, asic_id)
        if not ports:
            print('Failed to get port config', file=sys.stderr)
            sys.exit(1)
        deep_update(data, {'PORT': ports})

    for json_file in args.json:
        with open(json_file, 'r') as stream:
            deep_update(data, FormatConverter.to_deserialized(json.load(stream)))

    if args.minigraph != None:
        minigraph = args.minigraph
        if platform:
            if args.port_config != None:
                deep_update(data, parse_xml(minigraph, platform, args.port_config, asic_name=asic_name))
            else:
                deep_update(data, parse_xml(minigraph, platform, asic_name=asic_name))
        else:
            deep_update(data, parse_xml(minigraph, port_config_file=args.port_config, asic_name=asic_name))

    if args.device_description != None:
        deep_update(data, parse_device_desc_xml(args.device_description))

    for yaml_file in args.yaml:
        with open(yaml_file, 'r') as stream:
            if yaml.__version__ >= "5.1":
                additional_data = yaml.full_load(stream)
            else:
                additional_data = yaml.load(stream)
            deep_update(data, FormatConverter.to_deserialized(additional_data))

    if args.additional_data != None:
        deep_update(data, json.loads(args.additional_data))

    if args.from_db:
        if args.namespace is None:
            configdb = ConfigDBConnector(**db_kwargs)
        else:
            configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)

        configdb.connect()
        deep_update(data, FormatConverter.db_to_output(configdb.get_config()))


    # the minigraph file must be provided to get the mac address for backend asics
    if args.platform_info:
        asic_role = None
        if asic_name is not None:
            if args.minigraph is not None:
                asic_role = parse_asic_sub_role(args.minigraph, asic_name)

            if asic_role is not None and asic_role.lower() == "backend":
                mac = get_system_mac(namespace=asic_name)
            else:
                mac = get_system_mac()
        else:
            mac = get_system_mac()

        hardware_data = {'DEVICE_METADATA': {'localhost': {
            'platform': platform,
            'mac': mac,
            }}}
        # The ID needs to be passed to the SAI to identify the asic.
        if asic_name is not None:
            hardware_data['DEVICE_METADATA']['localhost'].update(asic_id=asic_id)
        deep_update(data, hardware_data)

    if args.template is not None:
        template_file = os.path.abspath(args.template)
        paths  = ['/', '/usr/share/sonic/templates', os.path.dirname(template_file)]
        if args.template_dir is not None:
            template_dir = os.path.abspath(args.template_dir)
            paths.append(template_dir)
        loader = jinja2.FileSystemLoader(paths)

        redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1'))
        env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc)
        env.filters['sort_by_port_index'] = sort_by_port_index
        env.filters['ipv4'] = is_ipv4
        env.filters['ipv6'] = is_ipv6
        env.filters['unique_name'] = unique_name
        env.filters['pfx_filter'] = pfx_filter
        env.filters['ip_network'] = ip_network
        for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']:
            env.filters[attr] = partial(prefix_attr, attr)
        template = env.get_template(template_file)
        print(template.render(sort_data(data)))

    if args.var != None:
        template = jinja2.Template('{{' + args.var + '}}')
        print(template.render(data))

    if args.var_json != None and args.var_json in data:
        if args.key != None:
            print(json.dumps(FormatConverter.to_serialized(data[args.var_json], args.key), indent=4, cls=minigraph_encoder))
        else:
            print(json.dumps(FormatConverter.to_serialized(data[args.var_json]), indent=4, cls=minigraph_encoder))

    if args.write_to_db:
        if args.namespace is None:
            configdb = ConfigDBConnector(**db_kwargs)
        else:
            configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)

        configdb.connect(False)
        configdb.mod_config(FormatConverter.output_to_db(data))

    if args.print_data:
        print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder))

    if args.preset != None:
        data = generate_sample_config(data, args.preset)
        print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder))


if __name__ == "__main__":
    main()