Introduce ConfigDB (#808)

* [cfggen] Support reading from and writing to configdb
* [bgp] Move bgp_admin_state to configdb, support dynamic admin state change
* [sonic-utilities] Adapt configDB for admin status, support config save and config load

dockers/docker-database/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-base
+FROM docker-config-engine
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -33,5 +33,6 @@ RUN sed -ri 's/^(save .*$)/# \1/g;
             ' /etc/redis/redis.conf
 
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["configdb-load.sh", "/usr/bin/"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-database/configdb-load.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Wait until redis starts
+while true; do
+    if [ `redis-cli ping` == "PONG" ]; then
+        break
+    fi
+    sleep 1
+done
+
+# If there is a config db dump file, load it
+if [ -r /etc/sonic/config_db.json ]; then
+    sonic-cfggen -j /etc/sonic/config_db.json --write-to-db
+fi
+
+echo -en "SELECT 4\nSET CONFIG_DB_INITIALIZED true" | redis-cli
+

dockers/docker-database/supervisord.conf

@@ -10,3 +10,12 @@ autostart=true
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[program:configdb-load.sh]
+command=/usr/bin/configdb-load.sh
+priority=2
+autostart=true
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+

dockers/docker-fpm-quagga/Dockerfile.j2

@@ -23,6 +23,7 @@ RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs
 
 COPY ["start.sh", "/usr/bin/"]
+COPY ["bgpcfgd", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
 COPY ["daemons", "/etc/quagga/"]

dockers/docker-fpm-quagga/bgpcfgd

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import sys
+import redis
+import subprocess
+import syslog
+from swsssdk import ConfigDBConnector
+
+# Returns BGP ASN as a string
+def _get_bgp_asn_from_minigraph():
+    # Get BGP ASN from minigraph
+    proc = subprocess.Popen(
+            ['sonic-cfggen', '-m', '/etc/sonic/minigraph.xml', '-v', 'minigraph_bgp_asn'],
+            stdout=subprocess.PIPE,
+            shell=False,
+            stderr=subprocess.STDOUT)
+    stdout = proc.communicate()[0]
+    proc.wait()
+    return stdout.rstrip('\n')
+
+def bgp_config(asn, ip, config):
+    syslog.syslog(syslog.LOG_INFO, '[bgp cfgd] value for {} changed to {}'.format(ip, config))
+    # Currently dynamic config is supported only for bgp admin status
+    if config.has_key('admin_status'):
+        command_mod = 'no ' if config['admin_status'] == 'up' else ''
+        command = "vtysh -c 'configure terminal' -c 'router bgp {}' -c '{}neighbor {} shutdown'".format(asn, command_mod, ip)
+        
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+        stdout = p.communicate()[0]
+        p.wait()
+        if p.returncode != 0:
+            syslog.syslog(syslog.LOG_ERR, '[bgp cfgd] command execution returned {}. Command: "{}", stdout: "{}"'.format(p.returncode, command, stdout))
+
+def main():
+    sub = ConfigDBConnector()
+    bgp_asn = _get_bgp_asn_from_minigraph()
+    handler = lambda table, key, data: bgp_config(bgp_asn, key, data)
+    sub.subscribe('BGP_NEIGHBOR', handler)
+    sub.connect()
+    sub.listen()
+
+main()

dockers/docker-fpm-quagga/start.sh

@@ -1,11 +1,8 @@
 #!/usr/bin/env bash
 
 mkdir -p /etc/quagga
-if [ -f /etc/sonic/bgp_admin.yml ]; then
-    sonic-cfggen -m /etc/sonic/minigraph.xml -y /etc/sonic/bgp_admin.yml -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/bgpd.conf.j2 > /etc/quagga/bgpd.conf
-else
-    sonic-cfggen -m /etc/sonic/minigraph.xml -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/bgpd.conf.j2 > /etc/quagga/bgpd.conf
-fi
+sonic-cfggen -m -d -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/bgpd.conf.j2 > /etc/quagga/bgpd.conf
+
 sonic-cfggen -m /etc/sonic/minigraph.xml -t /usr/share/sonic/templates/zebra.conf.j2 > /etc/quagga/zebra.conf
 
 sonic-cfggen -m /etc/sonic/minigraph.xml -t /usr/share/sonic/templates/isolate.j2 > /usr/sbin/bgp-isolate
@@ -21,6 +18,8 @@ echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status
 
 rm -f /var/run/rsyslogd.pid
 
+supervisorctl start bgpcfgd
+
 supervisorctl start rsyslogd
 
 # Quagga has its own monitor process, 'watchquagga'

dockers/docker-fpm-quagga/supervisord.conf

@@ -26,3 +26,12 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[program:bgpcfgd]
+command=/usr/bin/bgpcfgd
+priority=4
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+

files/build_templates/sonic_debian_extension.j2

@@ -132,8 +132,7 @@ sudo bash -c "echo dhcp_as_static=true >> $FILESYSTEM_ROOT/etc/sonic/updategraph
 sudo bash -c "echo enabled=false > $FILESYSTEM_ROOT/etc/sonic/updategraph.conf"
 {% endif %}
 {% if shutdown_bgp_on_start == "y" %}
-sudo bash -c "echo bgp_admin_state: > $FILESYSTEM_ROOT/etc/sonic/bgp_admin.yml"
-sudo bash -c "echo '  all: off' >> $FILESYSTEM_ROOT/etc/sonic/bgp_admin.yml"
+sudo bash -c "echo '{ \"bgp_admin_state\": { \"all\": false }  }' >> $FILESYSTEM_ROOT/etc/sonic/config_db.json"
 {% endif %}
 # Copy SNMP configuration files
 sudo cp $IMAGE_CONFIGS/snmp/snmp.yml $FILESYSTEM_ROOT/etc/sonic/

rules/docker-config-engine.mk

@@ -2,6 +2,7 @@
 
 DOCKER_CONFIG_ENGINE = docker-config-engine.gz
 $(DOCKER_CONFIG_ENGINE)_PATH = $(DOCKERS_PATH)/docker-config-engine
+$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SWSSSDK_PY2)
 $(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)
 $(DOCKER_CONFIG_ENGINE)_LOAD_DOCKERS += $(DOCKER_BASE)
 SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE)

rules/docker-database.mk

@@ -3,11 +3,12 @@
 DOCKER_DATABASE = docker-database.gz
 $(DOCKER_DATABASE)_PATH = $(DOCKERS_PATH)/docker-database
 $(DOCKER_DATABASE)_DEPENDS += $(REDIS_SERVER) $(REDIS_TOOLS)
-$(DOCKER_DATABASE)_LOAD_DOCKERS += $(DOCKER_BASE)
+$(DOCKER_DATABASE)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE)
 SONIC_DOCKER_IMAGES += $(DOCKER_DATABASE)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_DATABASE)
 
 $(DOCKER_DATABASE)_CONTAINER_NAME = database
 $(DOCKER_DATABASE)_RUN_OPT += --net=host --privileged -t
+$(DOCKER_DATABASE)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
 $(DOCKER_DATABASE)_BASE_IMAGE_FILES += redis-cli:/usr/bin/redis-cli

rules/docker-platform-monitor.mk

@@ -3,7 +3,6 @@
 DOCKER_PLATFORM_MONITOR = docker-platform-monitor.gz
 $(DOCKER_PLATFORM_MONITOR)_PATH = $(DOCKERS_PATH)/docker-platform-monitor
 $(DOCKER_PLATFORM_MONITOR)_DEPENDS += $(SONIC_LEDD)
-$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2)
 $(DOCKER_PLATFORM_MONITOR)_LOAD_DOCKERS = $(DOCKER_CONFIG_ENGINE)
 
 SONIC_DOCKER_IMAGES += $(DOCKER_PLATFORM_MONITOR)

rules/sonic-config.mk

@@ -2,5 +2,6 @@
 
 SONIC_CONFIG_ENGINE = sonic_config_engine-1.0-py2-none-any.whl
 $(SONIC_CONFIG_ENGINE)_SRC_PATH = $(SRC_PATH)/sonic-config-engine
+$(SONIC_CONFIG_ENGINE)_DEPENDS += $(SWSSSDK_PY2)
 $(SONIC_CONFIG_ENGINE)_PYTHON_VERSION = 2
 SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)

src/sonic-config-engine/sonic-cfggen

@@ -1,4 +1,19 @@
 #!/usr/bin/env python
+"""sonic-cfggen
+
+A tool to read SONiC config data from one or more of the following sources:
+minigraph file, config DB, json file(s), yaml files(s), command line input,
+and write the data into DB, print as json, or render a jinja2 config template.
+
+Examples:
+    Render template with minigraph:        
+        sonic-cfggen -m -t /usr/share/template/bgpd.conf.j2
+    Dump config DB content into json file: 
+        sonic-cfggen -d --print-data > db_dump.json
+    Load content of json file into config DB:
+        sonic-cfggen -j db_dump.json --write-to-db
+See usage string for detail description for arguments.
+"""
 
 import sys
 import os.path
@@ -12,6 +27,7 @@ from minigraph import parse_xml
 from minigraph import parse_device_desc_xml
 from sonic_platform import get_machine_info
 from sonic_platform import get_platform_info
+from swsssdk import ConfigDBConnector
 
 def is_ipv4(value):
     if not value:
@@ -46,18 +62,64 @@ def unique_name(l):
             new_list.append(item)
     return new_list
 
+
+class FormatConverter:
+    """Convert config DB based schema to legacy minigraph based schema for backward capability.
+We will move to DB schema and remove this class when the config templates are modified.
+
+TODO(taoyl): Current version of config db only supports BGP admin states.
+    All other configuration are still loaded from minigraph. Plan to remove 
+    minigraph and move everything into config db in a later commit.
+    """
+    @staticmethod
+    def db_to_output(db_data):
+        data_bgp_admin = {}
+        for table_name, content in db_data.iteritems():
+            if table_name == 'BGP_NEIGHBOR':
+                for key, value in content.iteritems():
+                    if value.has_key('admin_status'):
+                        data_bgp_admin[key] = (value['admin_status'] == 'up')
+            elif table_name == 'DEVICE_METADATA':
+                if content['localhost'].has_key('bgp_default_status'):
+                    data_bgp_admin['all'] = (content['localhost']['bgp_default_status'] == 'up')
+
+        output_data = {'bgp_admin_state': data_bgp_admin} if data_bgp_admin else {}
+        return output_data
+
+    @staticmethod
+    def output_to_db(output_data):
+        db_data = {}
+        for key, value in output_data.iteritems():
+            if key == 'bgp_admin_state':
+                for neighbor, state in value.iteritems():
+                    if neighbor == 'all':
+                        if not db_data.has_key('DEVICE_METADATA'):
+                            db_data['DEVICE_METADATA'] = {'localhost': {}}
+                        db_data['DEVICE_METADATA']['localhost']['bgp_default_status'] = 'up' if state else 'down'
+                    else:
+                        if not db_data.has_key('BGP_NEIGHBOR'):
+                            db_data['BGP_NEIGHBOR'] = {}
+                        if not db_data['BGP_NEIGHBOR'].has_key(neighbor):
+                            db_data['BGP_NEIGHBOR'][neighbor] = {}
+                        db_data['BGP_NEIGHBOR'][neighbor]['admin_status'] = 'up' if state else 'down'
+        return db_data
+
+
 def main():
     parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.")
     group = parser.add_mutually_exclusive_group()
-    group.add_argument("-m", "--minigraph", help="minigraph xml file")
+    group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml')
     group.add_argument("-M", "--device-description", help="device description xml file")
     parser.add_argument("-p", "--port-config", help="port config file, used with -m")
-    parser.add_argument("-y", "--yaml", help="yaml file that contains addtional variables", action='append', default=[])
+    parser.add_argument("-y", "--yaml", help="yaml file that contains additional variables", action='append', default=[])
+    parser.add_argument("-j", "--json", help="json file that contains additional variables", action='append', default=[])
     parser.add_argument("-a", "--additional-data", help="addition data, in json string")
+    parser.add_argument("-d", "--from-db", help="read config from configdb", action='store_true')
     group = parser.add_mutually_exclusive_group()
     group.add_argument("-t", "--template", help="render the data with the template file")
     group.add_argument("-v", "--var", help="print the value of a variable, support jinja2 expression")
     group.add_argument("--var-json", help="print the value of a variable, in json format")
+    group.add_argument("--write-to-db", help="write config into configdb", action='store_true')
     group.add_argument("--print-data", help="print all data", action='store_true')
     args = parser.parse_args()
 
@@ -90,8 +152,17 @@ def main():
             additional_data = yaml.load(stream)      
             data.update(additional_data)
 
+    for json_file in args.json:
+        with open(json_file, 'r') as stream:
+            data.update(json.load(stream))
+
     if args.additional_data != None:
         data.update(json.loads(args.additional_data))
+  
+    if args.from_db:
+        configdb = ConfigDBConnector()
+        configdb.connect()
+        data.update(FormatConverter.db_to_output(configdb.get_config()))
 
     if args.template != None:
         template_file = os.path.abspath(args.template)
@@ -109,8 +180,14 @@ def main():
     if args.var_json != None:
         print json.dumps(data[args.var_json], indent=4, cls=minigraph_encoder)
 
+    if args.write_to_db:
+        configdb = ConfigDBConnector()
+        configdb.connect(False)
+        configdb.set_config(FormatConverter.output_to_db(data))
+
     if args.print_data:
         print json.dumps(data, indent=4, cls=minigraph_encoder)
 
+
 if __name__ == "__main__":
     main()

src/sonic-py-swsssdk

@@ -1 +1 @@
-Subproject commit 4cf7a59a5ffa74784f8067484b0dbee51433c184
+Subproject commit 9b54b80f1783808c5ae2a30e189f24a7404a8c95

src/sonic-utilities

@@ -1 +1 @@
-Subproject commit 5d8f98eeae60f1b5c6c5d6ad7cb4c31019558efd
+Subproject commit 6c7e22362fbc05ba455e7e336e2a88430de0de18