[docker-frr]: bring quagga docker features to frr docker (#2870)

- use superviord to manage process in frr docker
- intro separated configuration mode for frr
- bring quagga configuration template to frr.

Signed-off-by: Guohan Lu <gulv@microsoft.com>

dockers/docker-fpm-frr/Dockerfile.j2

@@ -36,12 +36,8 @@ RUN apt-get autoclean -y
 RUN apt-get autoremove -y
 RUN rm -rf /debs ~/.cache
 
+COPY ["bgpcfgd", "start.sh", "/usr/bin/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
-COPY ["start.sh", "config.sh", "/usr/bin/"]
-COPY ["daemons", "/etc/frr/"]
-COPY ["daemons.conf", "/etc/frr/"]
-COPY ["vtysh.conf", "/etc/frr/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 
-ENTRYPOINT /usr/bin/config.sh   \
-        && /usr/bin/start.sh    \
-        && /bin/bash
+ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-fpm-frr/bgpcfgd

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+import sys
+import redis
+import subprocess
+import syslog
+from swsssdk import ConfigDBConnector
+
+class BGPConfigDaemon:
+
+    def __init__(self):
+        self.config_db = ConfigDBConnector()
+        self.config_db.connect()
+        self.bgp_asn = self.config_db.get_entry('DEVICE_METADATA', 'localhost')['bgp_asn']
+        self.bgp_neighbor = self.config_db.get_table('BGP_NEIGHBOR')
+
+    def __run_command(self, command):
+#        print command
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+        stdout = p.communicate()[0]
+        p.wait()
+        if p.returncode != 0:
+            syslog.syslog(syslog.LOG_ERR, '[bgp cfgd] command execution returned {}. Command: "{}", stdout: "{}"'.format(p.returncode, command, stdout))
+
+    def metadata_handler(self, key, data):
+        if key == 'localhost' and data.has_key('bgp_asn'):
+            if data['bgp_asn'] != self.bgp_asn:
+                syslog.syslog(syslog.LOG_INFO, '[bgp cfgd] ASN changed to {} from {}, restart BGP...'.format(data['bgp_asn'], self.bgp_asn))
+                self.__run_command("supervisorctl restart start.sh")
+                self.__run_command("service quagga restart")
+                self.bgp_asn = data['bgp_asn']
+
+    def bgp_handler(self, key, data):
+        syslog.syslog(syslog.LOG_INFO, '[bgp cfgd] value for {} changed to {}'.format(key, data))
+        if not data:
+            # Neighbor is deleted
+            command = "vtysh -c 'configure terminal' -c 'router bgp {}' -c 'no neighbor {}'".format(self.bgp_asn, key)
+            self.__run_command(command)
+            self.bgp_neighbor.pop(key)
+        else:
+            command = "vtysh -c 'configure terminal' -c 'router bgp {}' -c 'neighbor {} remote-as {}'".format(self.bgp_asn, key, data['asn'])
+            self.__run_command(command)
+            if data.has_key('name'):
+                command = "vtysh -c 'configure terminal' -c 'router bgp {}' -c 'neighbor {} description {}'".format(self.bgp_asn, key, data['name'])
+                self.__run_command(command)
+            if data.has_key('admin_status'):
+                command_mod = 'no ' if data['admin_status'] == 'up' else ''
+                command = "vtysh -c 'configure terminal' -c 'router bgp {}' -c '{}neighbor {} shutdown'".format(self.bgp_asn, command_mod, key)
+                self.__run_command(command)
+            self.bgp_neighbor[key] = data
+
+    def start(self):
+        self.config_db.subscribe('BGP_NEIGHBOR', 
+                lambda table, key, data: self.bgp_handler(key, data))
+        self.config_db.subscribe('DEVICE_METADATA',
+                lambda table, key, data: self.metadata_handler(key, data))
+        self.config_db.listen()
+
+
+def main():
+    daemon = BGPConfigDaemon()
+    daemon.start()
+
+if __name__ == "__main__":
+    main()

dockers/docker-fpm-frr/bgpd.conf.j2

@@ -0,0 +1,140 @@
+!
+{% block banner %}
+! =========== Managed by sonic-cfggen DO NOT edit manually! ====================
+! generated by templates/quagga/bgpd.conf.j2 with config DB data
+! file: bgpd.conf
+!
+{% endblock banner %}
+!
+{% block system_init %}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
+password zebra
+log syslog informational
+log facility local4
+! enable password {# {{ en_passwd }} TODO: param needed #}
+{% endblock system_init %}
+!
+{% if DEVICE_METADATA['localhost'].has_key('bgp_asn') %}
+{% block bgp_init %}
+!
+! bgp multiple-instance
+!
+route-map FROM_BGP_SPEAKER_V4 permit 10
+!
+route-map TO_BGP_SPEAKER_V4 deny 10
+!
+router bgp {{ DEVICE_METADATA['localhost']['bgp_asn'] }}
+  bgp log-neighbor-changes
+  bgp bestpath as-path multipath-relax
+  no bgp default ipv4-unicast
+  bgp graceful-restart restart-time 240
+  bgp graceful-restart
+{% for (name, prefix) in LOOPBACK_INTERFACE %}
+{% if prefix | ipv4 and name == 'Loopback0' %}
+  bgp router-id {{ prefix | ip }}
+{% endif %}
+{% endfor %}
+{# advertise loopback #}
+{% for (name, prefix) in LOOPBACK_INTERFACE %}
+{% if prefix | ipv4 and name == 'Loopback0' %}
+  network {{ prefix | ip }}/32
+{% elif prefix | ipv6 and name == 'Loopback0' %}
+  address-family ipv6
+    network {{ prefix | ip }}/64
+  exit-address-family
+{% endif %}
+{% endfor %}
+{% endblock bgp_init %}
+{% endif %}
+{% block vlan_advertisement %}
+{% for (name, prefix) in VLAN_INTERFACE %}
+{% if prefix | ipv4 %}
+  network {{ prefix }}
+{% elif prefix | ipv6 %}
+  address-family ipv6
+   network {{ prefix }}
+  exit-address-family
+{% endif %}
+{% endfor %}
+{% endblock vlan_advertisement %}
+{% block bgp_sessions %}
+{% for neighbor_addr, bgp_session in BGP_NEIGHBOR.iteritems() %}
+{% if bgp_session['asn'] | int != 0 %}
+  neighbor {{ neighbor_addr }} remote-as {{ bgp_session['asn'] }}
+  neighbor {{ neighbor_addr }} description {{ bgp_session['name'] }}
+{# set the bgp neighbor timers if they have not default values #}
+{% if     (bgp_session['keepalive'] is defined and bgp_session['keepalive'] | int != 60)
+      or  (bgp_session['holdtime'] is defined  and bgp_session['holdtime']  | int != 180) %}
+  neighbor {{ neighbor_addr }} timers {{ bgp_session['keepalive'] }} {{ bgp_session['holdtime'] }}
+{% endif %}
+{% if bgp_session.has_key('admin_status') and bgp_session['admin_status'] == 'down' or not bgp_session.has_key('admin_status') and DEVICE_METADATA['localhost'].has_key('default_bgp_status') and DEVICE_METADATA['localhost']['default_bgp_status'] == 'down' %}
+  neighbor {{ neighbor_addr }} shutdown
+{% endif %}
+{% if neighbor_addr | ipv4 %}
+  address-family ipv4
+{% if DEVICE_METADATA['localhost']['type'] == 'ToRRouter' %}
+    neighbor {{ neighbor_addr }} allowas-in 1
+{% endif %}
+    neighbor {{ neighbor_addr }} activate
+    neighbor {{ neighbor_addr }} soft-reconfiguration inbound
+    maximum-paths 64
+  exit-address-family
+{% endif %}
+{% if neighbor_addr | ipv6 %}
+  address-family ipv6
+{% if DEVICE_METADATA['localhost']['type'] == 'ToRRouter' %}
+    neighbor {{ neighbor_addr }} allowas-in 1
+{% endif %}
+    neighbor {{ neighbor_addr }} activate
+    neighbor {{ neighbor_addr }} soft-reconfiguration inbound
+    maximum-paths 64
+  exit-address-family
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endblock bgp_sessions %}
+{% block bgp_peers_with_range %}
+{% if BGP_PEER_RANGE %}
+{% for bgp_peer in BGP_PEER_RANGE.values() %}
+  neighbor {{ bgp_peer['name'] }} peer-group
+  neighbor {{ bgp_peer['name'] }} passive
+{% if bgp_peer['peer_asn'] is defined %}
+  neighbor {{ bgp_peer['name'] }} remote-as {{ bgp_peer['peer_asn'] }}
+{% else %}
+  neighbor {{ bgp_peer['name'] }} remote-as {{ deployment_id_asn_map[DEVICE_METADATA['localhost']['deployment_id']] }}
+{% endif %}
+  neighbor {{ bgp_peer['name'] }} ebgp-multihop 255
+  neighbor {{ bgp_peer['name'] }} soft-reconfiguration inbound
+{% if bgp_peer['src_address'] is defined %}
+  neighbor {{ bgp_peer['name'] }} update-source {{ bgp_peer['src_address'] | ip }}
+{% else %}
+{% for (name, prefix) in LOOPBACK_INTERFACE %}
+{% if name == 'Loopback1' %}
+  neighbor {{ bgp_peer['name'] }} update-source {{ prefix | ip }}
+{% endif %}
+{% endfor %}
+{% endif %}
+  neighbor {{ bgp_peer['name'] }} route-map FROM_BGP_SPEAKER_V4 in
+  neighbor {{ bgp_peer['name'] }} route-map TO_BGP_SPEAKER_V4 out
+{% for ip_range in bgp_peer['ip_range'] %}
+  bgp listen range {{ip_range}} peer-group {{ bgp_peer['name'] }}
+{% endfor %}
+  address-family ipv4
+    neighbor {{ bgp_peer['name'] }} activate
+    maximum-paths 64
+  exit-address-family
+  address-family ipv6
+    neighbor {{ bgp_peer['name'] }} activate
+    maximum-paths 64
+  exit-address-family
+{% endfor %}
+{% endif %}
+{% endblock bgp_peers_with_range %}
+!
+{% if DEVICE_METADATA['localhost'].has_key('bgp_asn') %}
+maximum-paths 64
+!
+route-map ISOLATE permit 10
+set as-path prepend {{ DEVICE_METADATA['localhost']['bgp_asn'] }}
+{% endif %}
+!

dockers/docker-fpm-frr/config.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-mkdir -p /etc/frr
-
-CONFIG_TYPE=`sonic-cfggen -d -v 'DEVICE_METADATA["localhost"]["docker_routing_config_mode"]'`
-
-if [ -z "$CONFIG_TYPE" ] || [ "$CONFIG_TYPE" == "unified" ]; then
-    sonic-cfggen -d -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/frr.conf.j2 >/etc/frr/frr.conf
-fi
-
-sonic-cfggen -d -t /usr/share/sonic/templates/isolate.j2 >/usr/sbin/bgp-isolate
-chown root:root /usr/sbin/bgp-isolate
-chmod 0755 /usr/sbin/bgp-isolate
-
-sonic-cfggen -d -t /usr/share/sonic/templates/unisolate.j2 >/usr/sbin/bgp-unisolate
-chown root:root /usr/sbin/bgp-unisolate
-chmod 0755 /usr/sbin/bgp-unisolate
-
-mkdir -p /var/sonic
-echo "# Config files managed by sonic-config-engine" >/var/sonic/config_status

dockers/docker-fpm-frr/daemons

@@ -1,65 +0,0 @@
-# This file tells the frr package which daemons to start.
-#
-# Sample configurations for these daemons can be found in
-# /usr/share/doc/frr/examples/.
-#
-# ATTENTION:
-#
-# When activation a daemon at the first time, a config file, even if it is
-# empty, has to be present *and* be owned by the user and group "frr", else
-# the daemon will not be started by /etc/init.d/frr. The permissions should
-# be u=rw,g=r,o=.
-# When using "vtysh" such a config file is also needed. It should be owned by
-# group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too.
-#
-# The watchfrr and zebra daemons are always started.
-#
-bgpd=yes
-ospfd=no
-ospf6d=no
-ripd=no
-ripngd=no
-isisd=no
-pimd=no
-ldpd=no
-nhrpd=no
-eigrpd=no
-babeld=no
-sharpd=no
-pbrd=no
-bfdd=no
-fabricd=no
-
-#
-# If this option is set the /etc/init.d/frr script automatically loads
-# the config via "vtysh -b" when the servers are started.
-# Check /etc/pam.d/frr if you intend to use "vtysh"!
-#
-vtysh_enable=yes
-zebra_options="  -A 127.0.0.1 -s 90000000 -M fpm"
-bgpd_options="   -A 127.0.0.1"
-ospfd_options="  -A 127.0.0.1"
-ospf6d_options=" -A ::1"
-ripd_options="   -A 127.0.0.1"
-ripngd_options=" -A ::1"
-isisd_options="  -A 127.0.0.1"
-pimd_options="   -A 127.0.0.1"
-ldpd_options="   -A 127.0.0.1"
-nhrpd_options="  -A 127.0.0.1"
-eigrpd_options=" -A 127.0.0.1"
-babeld_options=" -A 127.0.0.1"
-sharpd_options=" -A 127.0.0.1"
-pbrd_options="   -A 127.0.0.1"
-staticd_options="-A 127.0.0.1"
-bfdd_options="   -A 127.0.0.1"
-fabricd_options="-A 127.0.0.1"
-
-# The list of daemons to watch is automatically generated by the init script.
-watchfrr_options="-r '/usr/lib/frr/watchfrr.sh restart %s' -s '/usr/lib/frr/watchfrr.sh start %s' -k '/usr/lib/frr/watchfrr.sh stop %s'"
-
-# for debugging purposes, you can specify a "wrap" command to start instead
-# of starting the daemon directly, e.g. to use valgrind on ospfd:
-#   ospfd_wrap="/usr/bin/valgrind"
-# or you can use "all_wrap" for all daemons, e.g. to use perf record:
-#   all_wrap="/usr/bin/perf record --call-graph -"
-# the normal daemon command is added to this at the end.

dockers/docker-fpm-frr/daemons.conf

@@ -1 +0,0 @@
-# this file is deprecated, please use "daemons" instead.

dockers/docker-fpm-frr/start.sh

@@ -1,6 +1,41 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+mkdir -p /etc/frr
+
+CONFIG_TYPE=`sonic-cfggen -d -v 'DEVICE_METADATA["localhost"]["docker_routing_config_mode"]'`
+
+if [ -z "$CONFIG_TYPE" ] || [ "$CONFIG_TYPE" == "separated" ]; then
+    sonic-cfggen -d -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/bgpd.conf.j2 > /etc/frr/bgpd.conf
+    sonic-cfggen -d -t /usr/share/sonic/templates/zebra.conf.j2 > /etc/frr/zebra.conf
+    touch /etc/frr/vtysh.conf
+elif [ "$CONFIG_TYPE" == "unified" ]; then
+    sonic-cfggen -d -y /etc/sonic/deployment_id_asn_map.yml -t /usr/share/sonic/templates/frr.conf.j2 >/etc/frr/frr.conf
+    echo "service integrated-vtysh-config" > /etc/frr/vtysh.conf
+fi
+
+sonic-cfggen -d -t /usr/share/sonic/templates/isolate.j2 > /usr/sbin/bgp-isolate
+chown root:root /usr/sbin/bgp-isolate
+chmod 0755 /usr/sbin/bgp-isolate
+
+sonic-cfggen -d -t /usr/share/sonic/templates/unisolate.j2 > /usr/sbin/bgp-unisolate
+chown root:root /usr/sbin/bgp-unisolate
+chmod 0755 /usr/sbin/bgp-unisolate
+
+mkdir -p /var/sonic
+echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status
 
 rm -f /var/run/rsyslogd.pid
-service rsyslog start
-service frr start
-fpmsyncd &
+
+supervisorctl start rsyslogd
+
+supervisorctl start bgpcfgd
+
+# Start Quagga processes
+supervisorctl start zebra
+supervisorctl start bgpd
+
+if [ "$CONFIG_TYPE" == "unified" ]; then
+    supervisorctl start vtysh_b
+fi
+
+supervisorctl start fpmsyncd

dockers/docker-fpm-frr/supervisord.conf

@@ -0,0 +1,68 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[program:start.sh]
+command=/usr/bin/start.sh
+priority=1
+autostart=true
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:bgpcfgd]
+command=/usr/bin/bgpcfgd
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n
+priority=3
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:zebra]
+command=/usr/lib/frr/zebra -A 127.0.0.1 -s 90000000 -M fpm
+priority=4
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:bgpd]
+command=/usr/lib/frr/bgpd -A 127.0.0.1
+priority=5
+stopsignal=KILL
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:vtysh_b]
+command=/usr/bin/vtysh -b
+priority=6
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:fpmsyncd]
+command=fpmsyncd
+priority=6
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-fpm-frr/zebra.conf.j2

@@ -0,0 +1,78 @@
+!
+{% block banner %}
+! =========== Managed by sonic-cfggen DO NOT edit manually! ====================
+! generated by templates/quagga/zebra.conf.j2 using config DB data
+! file: zebra.conf
+!
+{% endblock banner %}
+!
+{% block sys_init %}
+hostname {{ DEVICE_METADATA['localhost']['hostname'] }}
+password zebra
+enable password zebra
+{% endblock sys_init %}
+!
+{% block interfaces %}
+! Enable link-detect (default disabled)
+{% for (name, prefix) in INTERFACE %}
+interface {{ name }}
+link-detect
+!
+{% endfor %}
+{% for pc in PORTCHANNEL %}
+interface {{ pc }}
+link-detect
+!
+{% endfor %}
+{% endblock interfaces %}
+!
+{% block default_route %}
+! set static default route to mgmt gateway as a backup to learned default
+{% for (name, prefix) in MGMT_INTERFACE %}
+{% if prefix | ipv4 %}
+ip route 0.0.0.0/0 {{ MGMT_INTERFACE[(name, prefix)]['gwaddr'] }} 200
+{% endif %}
+{% endfor %}
+{% endblock default_route %}
+!
+{% block source_loopback %}
+{% set lo_ipv4_addrs = [] %}
+{% set lo_ipv6_addrs = [] %}
+{% if LOOPBACK_INTERFACE %}
+{%   for (name, prefix) in LOOPBACK_INTERFACE %}
+{%     if name == 'Loopback0' %}
+{%       if prefix | ipv6 %}
+{%         if lo_ipv6_addrs.append(prefix) %}
+{%         endif %}
+{%       else %}
+{%         if lo_ipv4_addrs.append(prefix) %}
+{%         endif %}
+{%       endif %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+! Set ip source to loopback for bgp learned routes
+{% if lo_ipv4_addrs|length > 0 -%} 
+route-map RM_SET_SRC permit 10
+    set src {{ lo_ipv4_addrs[0] | ip }}
+!
+{% endif %}
+{% if lo_ipv6_addrs|length > 0 %} 
+route-map RM_SET_SRC6 permit 10
+    set src {{ lo_ipv6_addrs[0] | ip }}
+!
+{% endif %}
+ip protocol bgp route-map RM_SET_SRC
+!
+{% if lo_ipv6_addrs|length > 0 %}
+ipv6 protocol bgp route-map RM_SET_SRC6
+!
+{% endif %}
+{% endblock source_loopback %}
+!
+{% block logging %}
+log syslog informational
+log facility local4
+{% endblock logging %}
+!
+

files/build_templates/sonic_debian_extension.j2

@@ -337,8 +337,6 @@ sudo chmod 755 $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
 sudo mkdir $FILESYSTEM_ROOT/etc/sonic/frr
 sudo touch $FILESYSTEM_ROOT/etc/sonic/frr/frr.conf
 sudo touch $FILESYSTEM_ROOT/etc/sonic/frr/vtysh.conf
-sudo cp dockers/docker-fpm-frr/daemons.conf $FILESYSTEM_ROOT/etc/sonic/frr/
-sudo cp dockers/docker-fpm-frr/daemons $FILESYSTEM_ROOT/etc/sonic/frr/
 sudo chown -R $FRR_USER_UID:$FRR_USER_GID $FILESYSTEM_ROOT/etc/sonic/frr
 sudo chmod -R 640 $FILESYSTEM_ROOT/etc/sonic/frr/
 sudo chmod 750 $FILESYSTEM_ROOT/etc/sonic/frr