[cfggen] Add tool to translate openconfig acl into sonic format (#388)

* Build sonic-config-engine as whl instead of deb package
* Add tool to translate openconfig acl into sonic format

dockers/docker-config-engine/Dockerfile.j2

@@ -6,19 +6,42 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 
 # Dependencies for sonic-cfggen 
-RUN apt-get install -y python-lxml python-jinja2 python-netaddr python-ipaddr python-yaml 
+RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev
 
+RUN pip install --upgrade pip
+
+RUN pip install netaddr ipaddr jinja2 pyangbind
+
+{% if docker_config_engine_debs.strip() %}
 COPY \
 {% for deb in docker_config_engine_debs.split(' ') -%}
 debs/{{ deb }}{{' '}}
 {%- endfor -%}
 debs/
+{%- endif -%}
 
+{% if docker_config_engine_debs.strip() %}
 RUN dpkg -i \
 {% for deb in docker_config_engine_debs.split(' ') -%}
 debs/{{ deb }}{{' '}}
 {%- endfor %}
+{%- endif -%}
+
+{% if docker_config_engine_whls.strip() %}
+COPY \
+{% for whl in docker_config_engine_whls.split(' ') -%}
+python-wheels/{{ whl }}{{' '}}
+{%- endfor -%}
+python-wheels/
+{%- endif -%}
+
+{% if docker_config_engine_whls.strip() %}
+RUN pip install \
+{% for whl in docker_config_engine_whls.split(' ') -%}
+python-wheels/{{ whl }}{{' '}}
+{%- endfor %}
+{%- endif -%}
 
 ## Clean up
-RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN apt-get remove -y python-pip python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
-RUN rm -rf /debs
+RUN rm -rf /debs /python-wheels

files/build_templates/sonic_debian_extension.j2

@@ -55,16 +55,16 @@ sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/
 sudo mkdir -p $FILESYSTEM_ROOT/usr/share/sonic/templates/
 
 # Install dependencies for SONiC config engine 
-# TODO: pip-install instead of apt-get after config engine wrapped into a wheel, even better use pip implicitly installing dependencies
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
+    python-dev     \
     python-lxml    \
-    python-jinja2  \
+    python-yaml    \
-    python-netaddr \
+    python-bitarray
-    python-ipaddr  \
-    python-yaml
 
 # Install SONiC config engine
-sudo dpkg --root=$FILESYSTEM_ROOT -i {{config_engine}}
+CONFIG_ENGINE_WHL_NAME=`basename {{config_engine}}`
+sudo cp {{config_engine}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHL_NAME
+sudo chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHL_NAME
 
 # Install SONiC Utilities (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/python-sonic-utilities_*.deb || \
@@ -191,6 +191,10 @@ sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /sys || true
 sudo LANG=C chroot $FILESYSTEM_ROOT umount -lf /sys
 {% endif %}
 
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev 
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y 
+
 {% for file in installer_extra_files.split(' ') -%}
 {% if file.strip() -%}
 {% set src = file.split(':')[0] -%}

rules/docker-config-engine.mk

@@ -2,6 +2,6 @@
 
 DOCKER_CONFIG_ENGINE = docker-config-engine.gz
 $(DOCKER_CONFIG_ENGINE)_PATH = $(DOCKERS_PATH)/docker-config-engine
-$(DOCKER_CONFIG_ENGINE)_DEPENDS += $(SONIC_CONFIG_ENGINE)
+$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)
 $(DOCKER_CONFIG_ENGINE)_LOAD_DOCKERS += $(DOCKER_BASE)
 SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE)

rules/sonic-config.mk

@@ -1,5 +1,6 @@
 # sonic-config-engine package
 
-SONIC_CONFIG_ENGINE = python-sonic-config-engine_1.0-1_all.deb
+SONIC_CONFIG_ENGINE = sonic_config_engine-1.0-py2-none-any.whl
 $(SONIC_CONFIG_ENGINE)_SRC_PATH = $(SRC_PATH)/sonic-config-engine
-SONIC_PYTHON_STDEB_DEBS += $(SONIC_CONFIG_ENGINE)
+$(SONIC_CONFIG_ENGINE)_PYTHON_VERSION = 2
+SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)

slave.mk

@@ -280,6 +280,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .pl
 	sudo mount --bind $(PYTHON_WHEELS_PATH) $($*.gz_PATH)/python-wheels $(LOG)
 	# Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs
 	$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
+	$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_whls=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_PYTHON_WHEELS)))\n" | awk '!a[$$0]++'))
 	$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_PACKAGES)))\n" | awk '!a[$$0]++'))
 	j2 $($*.gz_PATH)/Dockerfile.j2 > $($*.gz_PATH)/Dockerfile
 	docker build --squash --no-cache -t $* $($*.gz_PATH) $(LOG)
@@ -299,7 +300,7 @@ $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TA
 ###############################################################################
 
 # targets for building installers with base image
-$(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : .platform onie-image.conf $$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)) $$(addprefix $(DEBS_PATH)/,$$($$*_INSTALLS)) $(addprefix $(DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL) $(IGB_DRIVER) $(SONIC_CONFIG_ENGINE) $(SONIC_DEVICE_DATA) $(SONIC_UTILS)) $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS))
+$(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : .platform onie-image.conf $$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)) $$(addprefix $(DEBS_PATH)/,$$($$*_INSTALLS)) $(addprefix $(DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL) $(IGB_DRIVER) $(SONIC_DEVICE_DATA) $(SONIC_UTILS)) $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))
 	$(HEADER)
 	## Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export initramfs_tools="$(DEBS_PATH)/$(INITRAMFS_TOOLS)"
@@ -308,7 +309,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : .platform
 	export installer_debs="$(addprefix $(DEBS_PATH)/,$($*_DEPENDS))"
 	export lazy_installer_debs="$(foreach deb, $($*_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(DEBS_PATH)/$(deb))))"
 	export installer_images="$(addprefix $(TARGET_PATH)/,$($*_DOCKERS))"
-	export config_engine="$(addprefix $(DEBS_PATH)/,$(SONIC_CONFIG_ENGINE))"
+	export config_engine="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))"
 	export image_type="$($*_IMAGE_TYPE)"
 	export sonicadmin_user="$(USERNAME)"
 	export sonic_asic_platform="$(CONFIGURED_PLATFORM)"

src/sonic-config-engine/setup.py

@@ -27,9 +27,9 @@ setup(name='sonic-config-engine',
       author='Taoyu Li',
       author_email='taoyl@microsoft.com',
       url='https://github.com/Azure/sonic-buildimage',
-      py_modules=['minigraph'],
+      py_modules=['minigraph', 'openconfig_acl'],
-      scripts=['sonic-cfggen'],
+      scripts=['sonic-cfggen', 'translate_acl'],
       data_files=get_platform_file_list(),
-      install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml'],
+      install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind'],
       test_suite='setup.get_test_suite',
      )

src/sonic-config-engine/translate_acl

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+import openconfig_acl 
+import pyangbind.lib.pybindJSON as pybindJSON
+
+import sys
+import os.path
+import json
+import argparse
+
+def dump_json(filename, data):
+    with open(filename, 'w') as outfile:
+        json.dump(data, outfile, indent=4, sort_keys=True, separators=(',', ':'))
+
+def generate_rule_json(table_name, rule, max_priority):
+    rule_idx = rule.config.sequence_id
+    rule_props = {}
+    rule_data = {}
+    rule_data["ACL_RULE_TABLE:"+table_name+":Rule_"+str(rule_idx)] = rule_props
+    rule_data["OP"] = "SET"
+    
+    rule_props["priority"] = max_priority - rule_idx
+    if rule.actions.config.forwarding_action == "ACCEPT":
+        rule_props["PACKET_ACTION"] = "FORWARD"
+    elif rule.actions.config.forwarding_action == "DROP":
+        rule_props["PACKET_ACTION"] = "DROP"
+    elif rule.actions.config.forwarding_action == "REJECT":
+        rule_props["PACKET_ACTION"] = "DROP"
+    else:
+        print "Unknown rule action %s in table %s, rule %d!" % (rule.actions.config.forwarding_action, table_name, rule_idx)
+        return {}
+    
+    ip_protocol_map = {
+        "IP_TCP" : 6,
+        "IP_ICMP" : 1,
+        "IP_UDP" : 17,
+        "IP_IGMP" : 2,
+        "IP_PIM" : 103,
+        "IP_RSVP" : 46,
+        "IP_GRE" : 47,
+        "IP_AUTH" : 51,
+        "IP_L2TP" : 115
+        }
+
+    if not rule.ip.config.protocol:
+        pass
+    elif ip_protocol_map.has_key(rule.ip.config.protocol):
+        rule_props["IP_PROTOCOL"] = ip_protocol_map[rule.ip.config.protocol]
+    else:
+        try:
+            int(rule.ip.config.protocol)
+        except:
+            print "Unknown rule protocol %s in table %s, rule %d!" % (rule.ip.config.protocol, table_name, rule_idx)
+            return {}
+        else:
+            rule_props["IP_PROTOCOL"] = rule.ip.config.protocol
+    
+    if rule.ip.config.source_ip_address != "":
+        rule_props["SRC_IP"] = rule.ip.config.source_ip_address
+    if rule.ip.config.destination_ip_address != "":
+        rule_props["DST_IP"] = rule.ip.config.destination_ip_address
+
+    if rule.transport.config.source_port == "":
+        pass
+    elif str(rule.transport.config.source_port).find("..") < 0:
+        rule_props["L4_SRC_PORT"] = rule.transport.config.source_port
+    else:
+        rule_props["L4_SRC_PORT_RANGE"] = str(rule.transport.config.source_port).replace("..", "-")
+
+    if rule.transport.config.destination_port == "":
+        pass
+    elif str(rule.transport.config.destination_port).find("..") < 0:
+        rule_props["L4_DST_PORT"] = rule.transport.config.destination_port
+    else:
+        rule_props["L4_DST_PORT_RANGE"] = str(rule.transport.config.destination_port).replace("..", "-")
+    
+    tcp_flags = 0x00;
+    for flag in rule.transport.config.tcp_flags:
+        if flag == "TCP_FIN":
+            tcp_flags = tcp_flags | 0x01
+        if flag == "TCP_SYN":
+            tcp_flags = tcp_flags | 0x02
+        if flag == "TCP_RST":
+            tcp_flags = tcp_flags | 0x04
+        if flag == "TCP_PSH":
+            tcp_flags = tcp_flags | 0x08
+        if flag == "TCP_ACK":
+            tcp_flags = tcp_flags | 0x10
+        if flag == "TCP_URG":
+            tcp_flags = tcp_flags | 0x20
+        if flag == "TCP_ECE":
+            tcp_flags = tcp_flags | 0x40
+        if flag == "TCP_CWR":
+            tcp_flags = tcp_flags | 0x80
+    if tcp_flags != 0x00:
+        rule_props["TCP_FLAGS"] = '0x{:02x}'.format(tcp_flags)
+    return rule_data
+
+def generate_table_json(aclset, aclname, port, max_priority, output_path='.'):
+    table_name = aclname.replace(" ", "_")
+    #table_name = generate_random_table_name()
+
+    table_props = {}
+    table_props["policy_desc"] = table_name
+    table_props["type"] = "L3"
+    table_props["ports"] = port
+
+    table_data = [{}]
+    table_data[0]["ACL_TABLE:"+table_name] = table_props
+    table_data[0]["OP"] = "SET"
+    dump_json(os.path.join(output_path, "table_"+table_name+".json"), table_data)
+
+    rule_data = []
+    for aclentryname in aclset.acl_entries.acl_entry:
+        aclentry = aclset.acl_entries.acl_entry[aclentryname]
+        rule_props = generate_rule_json(table_name, aclentry, max_priority)
+        if rule_props:
+            rule_data.append(rule_props)
+
+    dump_json(os.path.join(output_path, "rules_for_"+table_name+".json"), rule_data)
+
+def translate_acl(filename, output_path, port, max_priority):
+    yang_acl = pybindJSON.load(filename, openconfig_acl, "openconfig_acl")
+    for aclsetname in yang_acl.acl.acl_sets.acl_set:
+         aclset = yang_acl.acl.acl_sets.acl_set[aclsetname]
+         generate_table_json(aclset, aclsetname, port, max_priority, output_path)
+    return
+
+def main():
+    parser = argparse.ArgumentParser(description="Translate openconfig ACL json into SONiC ACL jsons")
+    parser.add_argument('input', metavar='INPUT', help='input json file in openconfig format')
+    parser.add_argument('-p', '--port', default='Ethernet0', help='the port(s) that this ACL is binding to')
+    parser.add_argument('-m', '--max-priority', type=int, default=10000, help='the priority number of the first rule in ACL entries')
+    parser.add_argument('-o', '--output-path', default='.', help='output directory where SONiC ACL jsons will be generated')
+    args = parser.parse_args()
+    translate_acl(args.input, args.output_path, args.port, args.max_priority)
+
+if __name__ == "__main__":
+    main()