From 4d07bbbec6b4826ae5383c08370cc07578d91dba Mon Sep 17 00:00:00 2001 From: arlakshm <55814491+arlakshm@users.noreply.github.com> Date: Thu, 10 Jun 2021 12:03:33 -0700 Subject: [PATCH] [Yang][cfggen] update sonic-cfggen to generate config_db from Yang data (#7712) Why I did it This PR adds changes in sonic-config-engine to consume configuration data in SONiC Yang schema and generate config_db entries How I did it Add a new file sonic_yang_cfg_generator . This file has the functions to parse yang data json and convert them in config_db json format. Validate the converted config_db entries to make sure all the dependencies and constraints are met. Add a new option -Y to the sonic-cfggen command for this purpose Add unit tests This capability is support only in sonic-config-engine Python3 package only --- .../build_templates/sonic_debian_extension.j2 | 31 +- rules/docker-config-engine-buster.mk | 10 +- rules/sonic-config.mk | 9 +- rules/sonic_bgpcfgd.mk | 9 +- src/sonic-config-engine/setup.py | 27 +- src/sonic-config-engine/sonic-cfggen | 16 +- .../sonic_yang_cfg_generator.py | 60 +++ .../tests/test_cfggen_from_yang.py | 267 ++++++++++++ .../tests/test_yang_data.json | 381 ++++++++++++++++++ src/sonic-yang-mgmt/sonic_yang_ext.py | 22 +- .../libyang-python-tests/test_sonic_yang.py | 6 +- 11 files changed, 800 insertions(+), 38 deletions(-) create mode 100644 src/sonic-config-engine/sonic_yang_cfg_generator.py create mode 100644 src/sonic-config-engine/tests/test_cfggen_from_yang.py create mode 100644 src/sonic-config-engine/tests/test_yang_data.json diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 2285d65603..328fd9f8b8 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -164,6 +164,22 @@ if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install libxslt-dev libz-dev fi +# Install sonic-yang-models Python 3 package, install dependencies +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb +SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}}) +sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME + +# Install sonic-yang-mgmt Python3 package +SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}}) +sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME + # Install SONiC config engine Python 2 package CONFIG_ENGINE_PY2_WHEEL_NAME=$(basename {{config_engine_py2_wheel_path}}) sudo cp {{config_engine_py2_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY2_WHEEL_NAME @@ -184,21 +200,6 @@ sudo cp {{config_engine_py3_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY3_WHE sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $CONFIG_ENGINE_PY3_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY3_WHEEL_NAME -# Install sonic-yang-models py3 package, install dependencies -sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb -sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb -sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb -sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb -SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}}) -sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME -sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME - -# Install sonic-yang-mgmt Python3 package -SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}}) -sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME -sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME # Install sonic-platform-common Python 2 package PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}}) diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index f7232b23d6..6dc1126530 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -3,9 +3,15 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz $(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster -$(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) +$(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) \ + $(LIBYANG) \ + $(LIBYANG_CPP) \ + $(LIBYANG_PY3) \ + $(PYTHON3_SWSSCOMMON) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3) -$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) +$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \ + $(SONIC_YANG_MGMT_PY3) \ + $(SONIC_YANG_MODELS_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) diff --git a/rules/sonic-config.mk b/rules/sonic-config.mk index e8e59fc1b9..373604e965 100644 --- a/rules/sonic-config.mk +++ b/rules/sonic-config.mk @@ -11,8 +11,13 @@ SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) SONIC_CONFIG_ENGINE_PY3 = sonic_config_engine-1.0-py3-none-any.whl $(SONIC_CONFIG_ENGINE_PY3)_SRC_PATH = $(SRC_PATH)/sonic-config-engine -$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) -$(SONIC_CONFIG_ENGINE_PY3)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON) +$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \ + $(SONIC_YANG_MGMT_PY3) \ + $(SONIC_YANG_MODELS_PY3) +$(SONIC_CONFIG_ENGINE_PY3)_DEBS_DEPENDS += $(LIBYANG) \ + $(LIBYANG_CPP) \ + $(LIBYANG_PY3) \ + $(PYTHON3_SWSSCOMMON) # Synthetic dependency to avoid building the Python 2 and 3 packages # simultaneously and any potential conflicts which may arise $(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY2) diff --git a/rules/sonic_bgpcfgd.mk b/rules/sonic_bgpcfgd.mk index 3e1395842b..9abab06800 100644 --- a/rules/sonic_bgpcfgd.mk +++ b/rules/sonic_bgpcfgd.mk @@ -7,7 +7,12 @@ $(SONIC_BGPCFGD)_SRC_PATH = $(SRC_PATH)/sonic-bgpcfgd # as part of its unit tests. # TODO: Refactor unit tests so that these dependencies are not needed -$(SONIC_BGPCFGD)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY3) -$(SONIC_BGPCFGD)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON) +$(SONIC_BGPCFGD)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY3) \ + $(SONIC_YANG_MGMT_PY3) \ + $(SONIC_YANG_MODELS_PY3) +$(SONIC_BGPCFGD)_DEBS_DEPENDS += $(LIBYANG) \ + $(LIBYANG_CPP) \ + $(LIBYANG_PY3) \ + $(PYTHON3_SWSSCOMMON) $(SONIC_BGPCFGD)_PYTHON_VERSION = 3 SONIC_PYTHON_WHEELS += $(SONIC_BGPCFGD) diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index 3df049a365..9e98ff607b 100644 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -20,7 +20,9 @@ if sys.version_info.major == 3: # Python3 has enum module and so pyangbind should be installed outside # dependencies section of setuptools followed by uninstall of enum43 # 'pyangbind==0.8.1', - 'Jinja2>=2.10' + 'Jinja2>=2.10', + 'sonic-yang-mgmt>=1.0', + 'sonic-yang-models>=1.0' ] else: # Python 2-only dependencies @@ -34,6 +36,20 @@ else: 'importlib-resources==3.3.1' # importlib-resources v4.0.0 was released 2020-12-23 and drops support for Python 2 ] +# Common modules for python2 and python3 +py_modules = [ + 'config_samples', + 'lazy_re', + 'minigraph', + 'openconfig_acl', + 'portconfig', + 'redis_bcc', +] +if sys.version_info.major == 3: + # Python 3-only modules + py_modules += [ + 'sonic_yang_cfg_generator' + ] setup( name = 'sonic-config-engine', @@ -42,14 +58,7 @@ setup( author = 'Taoyu Li', author_email = 'taoyl@microsoft.com', url = 'https://github.com/Azure/sonic-buildimage', - py_modules = [ - 'config_samples', - 'lazy_re', - 'minigraph', - 'openconfig_acl', - 'portconfig', - 'redis_bcc', - ], + py_modules = py_modules, scripts = [ 'sonic-cfggen', ], diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index c0d16ea969..26692399d9 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -48,9 +48,11 @@ from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBCo PY3x = sys.version_info >= (3, 0) -#TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x +# TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x +# TODO: Remove the import SonicYangCfgDbGenerator once SONiC moves to python3.x if PY3x: from io import IOBase + from sonic_yang_cfg_generator import SonicYangCfgDbGenerator STR_TYPE = str FILE_TYPE = IOBase else: @@ -258,6 +260,7 @@ def main(): parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.") group = parser.add_mutually_exclusive_group() group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml') + group.add_argument("-Y", "--yang", help="yang data json file", nargs='?', const='/etc/sonic/config_yang.json') group.add_argument("-M", "--device-description", help="device description xml file") group.add_argument("-k", "--hwsku", help="HwSKU") parser.add_argument("-n", "--namespace", help="namespace name", nargs='?', const=None, default=None) @@ -325,6 +328,17 @@ def main(): _process_json(args, data) + if args.yang is not None: + #TODO: Remove this check onces SONiC moves to python3.x + if PY3x: + yang_file = args.yang + config_db_json = SonicYangCfgDbGenerator().generate_config( + yang_data_file=yang_file) + deep_update(data, config_db_json) + else: + print('-Y/--yang option is not available in Python2', file=sys.stderr) + sys.exit(1) + if args.minigraph is not None: minigraph = args.minigraph if platform: diff --git a/src/sonic-config-engine/sonic_yang_cfg_generator.py b/src/sonic-config-engine/sonic_yang_cfg_generator.py new file mode 100644 index 0000000000..4702e65571 --- /dev/null +++ b/src/sonic-config-engine/sonic_yang_cfg_generator.py @@ -0,0 +1,60 @@ +"""sonic_yang_cfg_generator +version_added: "1.0" +author: Arvindsrinivasan Lakshmi naraismhan (arlakshm@microsoft.com) +short_description: Parse sonic_yang data and generate the config_db entries +""" + +from __future__ import print_function + +import json +import sys + + +import sonic_yang +# TODO: Remove this once we no longer support Python 2 +if sys.version_info.major == 3: + UNICODE_TYPE = str +else: + UNICODE_TYPE = unicode + +YANG_MODELS_DIR = "/usr/local/yang-models" +DEFAULT_YANG_DATA_FILE = "/etc/sonic/config_yang.json" + + +class SonicYangCfgDbGenerator: + + def __init__(self, yang_models_dir=YANG_MODELS_DIR): + self.yang_models_dir = yang_models_dir + self.yang_parser = sonic_yang.SonicYang(self.yang_models_dir) + self.yang_parser.loadYangModel() + + def get_config_db_from_yang_data(self, + yang_data_file=DEFAULT_YANG_DATA_FILE): + self.yang_data_file = yang_data_file + config_db_json = dict() + with open(self.yang_data_file, "r") as yang_file: + try: + self.yang_data = json.load(yang_file) + config_db_json = self.yang_parser.XlateYangToConfigDB( + yang_data=self.yang_data) + except json.JSONDecodeError as e: + print("Unable to parse Yang data file {} Error: {}".format( + yang_data_file, e)) + return config_db_json + + def validate_config_db_json(self, config_db_json): + self.yang_parser.loadData(configdbJson=config_db_json) + try: + self.yang_parser.validate_data_tree() + return True + except sonic_yang.SonicYangException as e: + print("yang data in {} is not valid".format(self.yang_data_file)) + return False + + def generate_config(self, yang_data_file=DEFAULT_YANG_DATA_FILE): + config_db_json = self.get_config_db_from_yang_data( + yang_data_file=yang_data_file) + if self.validate_config_db_json(config_db_json): + return config_db_json + else: + return {} diff --git a/src/sonic-config-engine/tests/test_cfggen_from_yang.py b/src/sonic-config-engine/tests/test_cfggen_from_yang.py new file mode 100644 index 0000000000..d673e1e206 --- /dev/null +++ b/src/sonic-config-engine/tests/test_cfggen_from_yang.py @@ -0,0 +1,267 @@ +import json +import pytest +import subprocess +import os + +import tests.common_utils as utils + + +#TODO: Remove this fixuture once SONiC moves to python3.x +@pytest.fixture(scope="class") +def is_test_supported(): + if not utils.PY3x: + pytest.skip('module not support in python2') + else: + pass + + +@pytest.mark.usefixtures("is_test_supported") +class TestCfgGen(object): + + @pytest.fixture(autouse=True) + def setup_teardown(self): + self.test_dir = os.path.dirname(os.path.realpath(__file__)) + self.script_file = utils.PYTHON_INTERPRETTER + ' ' + os.path.join( + self.test_dir, '..', 'sonic-cfggen') + self.sample_yang_file = os.path.join(self.test_dir, + 'test_yang_data.json') + + def run_script(self, arg, check_stderr=False): + print('\n Running sonic-cfggen ' + arg) + if check_stderr: + output = subprocess.check_output(self.script_file + ' ' + arg, + stderr=subprocess.STDOUT, + shell=True) + else: + output = subprocess.check_output(self.script_file + ' ' + arg, + shell=True) + + if utils.PY3x: + output = output.decode() + + linecount = output.strip().count('\n') + if linecount <= 0: + print(' Output: ' + output.strip()) + else: + print(' Output: ({0} lines, {1} bytes)'.format( + linecount + 1, len(output))) + return output + + def run_diff(self, file1, file2): + return subprocess.check_output('diff -u {} {} || true'.format( + file1, file2), + shell=True) + + def run_script_with_yang_arg(self, arg, check_stderr=False): + args = "-Y {} {}".format(self.sample_yang_file, arg) + return self.run_script(arg=args, check_stderr=check_stderr) + + def test_print_data(self): + arg = "--print-data" + output = self.run_script_with_yang_arg(arg) + assert len(output.strip()) > 0 + + + def test_jinja_expression(self, expected_router_type='LeafRouter'): + arg = " -v \"DEVICE_METADATA[\'localhost\'][\'type\']\" " + output = self.run_script_with_yang_arg(arg) + assert output.strip() == expected_router_type + + def test_hwsku(self): + arg = "-v \"DEVICE_METADATA[\'localhost\'][\'hwsku\']\" " + output = self.run_script_with_yang_arg(arg) + assert output.strip() == "Force10-S6000" + + def test_device_metadata(self): + arg = "--var-json \"DEVICE_METADATA\" " + output = json.loads(self.run_script_with_yang_arg(arg)) + assert (output['localhost'] == {\ + 'bgp_asn': '65100', + 'docker_routing_config_mode': 'unified', + 'hostname': 'sonic', + 'hwsku': 'Force10-S6000', + 'platform': 'x86_64-kvm_x86_64-r0', + 'type': 'LeafRouter' + }) + + + def test_port_table(self): + arg = "--var-json \"PORT\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == \ + {'Ethernet0': {'admin_status': 'up', 'alias': 'eth0', 'description': 'Ethernet0', 'fec': 'rs', 'lanes': '65, 66', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet4': {'admin_status': 'up', 'alias': 'eth4', 'description': 'Ethernet4', 'fec': 'rs', 'lanes': '67, 68', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet8': {'admin_status': 'up', 'alias': 'eth8', 'description': 'Ethernet8', 'fec': 'rs', 'lanes': '69, 70', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet12': {'admin_status': 'up', 'alias': 'eth12', 'description': 'Ethernet12', 'fec': 'rs', 'lanes': '71, 72', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet16': {'admin_status': 'up', 'alias': 'eth16', 'description': 'Ethernet16', 'fec': 'rs', 'lanes': '73, 74', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet20': {'admin_status': 'up', 'alias': 'eth20', 'description': 'Ethernet20', 'fec': 'rs', 'lanes': '75,76', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet24': {'admin_status': 'up', 'alias': 'eth24', 'description': 'Ethernet24', 'fec': 'rs', 'lanes': '77,78', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'}, + 'Ethernet28': {'admin_status': 'up', 'alias': 'eth28', 'description': 'Ethernet28', 'fec': 'rs', 'lanes': '79,80', 'mtu': '9100', 'pfc_asym': 'on', 'speed': '40000'} + }) + + def test_portchannel_table(self): + arg = "--var-json \"PORTCHANNEL\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == \ + {'PortChannel1001': {'admin_status': 'up', + 'lacp_key': 'auto', + 'members': ['Ethernet0', 'Ethernet4'], + 'min_links': '1', + 'mtu': '9100'}, + 'PortChannel1002': {'admin_status': 'up', + 'lacp_key': 'auto', + 'members': ['Ethernet16', 'Ethernet20'], + 'min_links': '1', + 'mtu': '9100'}}) + + def test_portchannel_member_table(self): + arg = "--var-json \"PORTCHANNEL_MEMBER\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output ==\ + { "PortChannel1001|Ethernet0": {}, + "PortChannel1001|Ethernet4": {}, + "PortChannel1002|Ethernet16": {}, + "PortChannel1002|Ethernet20": {} + }) + + def test_interface_table(self): + arg = "--var-json \"INTERFACE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output =={\ + "Ethernet8": {}, + "Ethernet12": {}, + "Ethernet8|10.0.0.8/31": { + "family": "IPv4", + "scope": "global" + }, + "Ethernet8|fc::01/126": { + "family": "IPv6", + "scope": "global" + }, + "Ethernet12|10.0.0.12/31": { + "family": "IPv4", + "scope": "global" + }, + "Ethernet12|fd::01/126": { + "family": "IPv6", + "scope": "global" + } + }) + + def test_portchannel_interface_table(self): + arg = "--var-json \"PORTCHANNEL_INTERFACE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output =={\ + "PortChannel1001|10.0.0.1/31": {}, + "PortChannel1002|10.0.0.60/31": {} + }) + + def test_loopback_table(self): + arg = "--var-json \"LOOPBACK_INTERFACE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "Loopback0": {}, + "Loopback0|aa::01/64": { + "family": "IPv6", + "scope": "global" + }, + "Loopback0|10.1.0.32/32": { + "family": "IPv4", + "scope": "global" + } + }) + + def test_acl_table(self): + arg = "--var-json \"ACL_TABLE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + 'DATAACL': {'policy_desc': 'DATAACL', 'ports': ['PortChannel1001','PortChannel1002'], 'stage': 'ingress', 'type': 'L3'}, + 'EVERFLOW': {'policy_desc': 'EVERFLOW', 'ports': ['PortChannel1001','PortChannel1002'], 'stage': 'ingress', 'type': 'MIRROR'}, + 'EVERFLOWV6': {'policy_desc': 'EVERFLOWV6', 'ports': ['PortChannel1001','PortChannel1002'], 'stage': 'ingress', 'type': 'MIRRORV6'}, + 'SNMP_ACL': {'policy_desc': 'SNMP_ACL', 'services': ['SNMP'], 'stage': 'ingress', 'type': 'CTRLPLANE'}, + 'SSH_ONLY': {'policy_desc': 'SSH_ONLY', 'services': ['SSH'], 'stage': 'ingress', 'type': 'CTRLPLANE'}}) + + def test_acl_rule(self): + arg = "--var-json \"ACL_RULE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "DATAACL|Rule1": { + "DST_IP": "192.168.1.1/24", + "SRC_IP": "10.10.1.1/16", + "IP_TYPE": "IPV4", + "PACKET_ACTION": "DROP", + "PRIORITY": "100" + }, + "EVERFLOW|Rule2": { + "DST_IP": "192.169.10.1/32", + "SRC_IP": "10.10.1.1/16", + "IP_TYPE": "IPV4" + } + }) + + def test_vlan_table(self): + arg = "--var-json \"VLAN\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "192.169.1.1", + "198.169.1.1", + "199.169.1.2" + ], + "mtu": "9100", + "vlanid": "100" + } + }) + + def test_vlan_interface(self): + arg = "--var-json \"VLAN_INTERFACE\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "Vlan100": {}, + "Vlan100|bb::01/64": { + "family": "IPv6", + "scope": "global" + }, + "Vlan100|10.35.61.1/24": { + "family": "IPv4", + "scope": "global" + } + }) + + def test_vlan_member(self): + arg = "--var-json \"VLAN_MEMBER\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "Vlan100|Ethernet24": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet28": { + "tagging_mode": "untagged" + } + }) + + def test_vlan_crm(self): + arg = "--var-json \"CRM\"" + output = json.loads(self.run_script_with_yang_arg(arg)) + assert(output == {\ + "Config": { + "acl_counter_high_threshold": "85", + "acl_counter_low_threshold": "25", + "acl_counter_threshold_type": "used", + "ipv4_route_high_threshold": "90", + "ipv4_route_low_threshold": "10", + "ipv4_route_threshold_type": "used", + "ipv6_route_high_threshold": "90", + "ipv6_route_low_threshold": "10", + "ipv6_route_threshold_type": "used", + "ipv4_neighbor_high_threshold": "85", + "ipv4_neighbor_low_threshold": "25", + "ipv4_neighbor_threshold_type": "used", + "ipv6_neighbor_high_threshold": "90", + "ipv6_neighbor_low_threshold": "10", + "ipv6_neighbor_threshold_type": "used" + } + }) diff --git a/src/sonic-config-engine/tests/test_yang_data.json b/src/sonic-config-engine/tests/test_yang_data.json new file mode 100644 index 0000000000..3a28872317 --- /dev/null +++ b/src/sonic-config-engine/tests/test_yang_data.json @@ -0,0 +1,381 @@ +{ + "sonic-port:sonic-port": { + "sonic-port:PORT": { + "PORT_LIST": [ + { + "admin_status": "up", + "alias": "eth0", + "description": "Ethernet0", + "fec": "rs", + "lanes": "65, 66", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet0", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth4", + "description": "Ethernet4", + "fec": "rs", + "lanes": "67, 68", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet4", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth8", + "description": "Ethernet8", + "fec": "rs", + "lanes": "69, 70", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet8", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth12", + "description": "Ethernet12", + "fec": "rs", + "lanes": "71, 72", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet12", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth16", + "description": "Ethernet16", + "fec": "rs", + "lanes": "73, 74", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet16", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth20", + "description": "Ethernet20", + "fec": "rs", + "lanes": "75,76", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet20", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth24", + "description": "Ethernet24", + "fec": "rs", + "lanes": "77,78", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet24", + "speed": 40000 + }, + { + "admin_status": "up", + "alias": "eth28", + "description": "Ethernet28", + "fec": "rs", + "lanes": "79,80", + "mtu": 9100, + "pfc_asym": "on", + "name": "Ethernet28", + "speed": 40000 + } + ] + } + }, + "sonic-portchannel:sonic-portchannel": { + "sonic-portchannel:PORTCHANNEL": { + "PORTCHANNEL_LIST": [ + { + "admin_status": "up", + "members": [ + "Ethernet0", + "Ethernet4" + ], + "min_links": "1", + "mtu": "9100", + "lacp_key": "auto", + "name": "PortChannel1001" + }, + { + "admin_status": "up", + "members": [ + "Ethernet16", + "Ethernet20" + ], + "min_links": "1", + "mtu": "9100", + "lacp_key": "auto", + "name": "PortChannel1002" + } + ] + }, + "sonic-portchannel:PORTCHANNEL_MEMBER": { + "PORTCHANNEL_MEMBER_LIST": [ + { + "name": "PortChannel1001", + "port": "Ethernet0" + }, + { + "name": "PortChannel1001", + "port": "Ethernet4" + }, + { + "name": "PortChannel1002", + "port": "Ethernet16" + }, + { + "name": "PortChannel1002", + "port": "Ethernet20" + } + ] + }, + "sonic-portchannel:PORTCHANNEL_INTERFACE": { + "PORTCHANNEL_INTERFACE_IPPREFIX_LIST": [ + { + "name": "PortChannel1001", + "ip_prefix": "10.0.0.1/31" + }, + { + "name": "PortChannel1002", + "ip_prefix": "10.0.0.60/31" + } + ] + } + }, + "sonic-device_metadata:sonic-device_metadata": { + "sonic-device_metadata:DEVICE_METADATA": { + "localhost": { + "bgp_asn": "65100", + "docker_routing_config_mode": "unified", + "hostname": "sonic", + "hwsku": "Force10-S6000", + "type": "LeafRouter", + "platform": "x86_64-kvm_x86_64-r0" + } + } + }, + "sonic-interface:sonic-interface": { + "sonic-interface:INTERFACE": { + "INTERFACE_IPPREFIX_LIST": [ + { + "family": "IPv4", + "ip-prefix": "10.0.0.8/31", + "name": "Ethernet8", + "scope": "global" + }, + { + "family": "IPv6", + "ip-prefix": "fc::01/126", + "name": "Ethernet8", + "scope": "global" + }, + { + "family": "IPv4", + "ip-prefix": "10.0.0.12/31", + "name": "Ethernet12", + "scope": "global" + }, + { + "family": "IPv6", + "ip-prefix": "fd::01/126", + "name": "Ethernet12", + "scope": "global" + } + ], + "INTERFACE_LIST": [ + { + "name": "Ethernet8" + }, + { + "name": "Ethernet12" + } + ] + } + }, + "sonic-loopback-interface:sonic-loopback-interface": { + "sonic-loopback-interface:LOOPBACK_INTERFACE": { + "LOOPBACK_INTERFACE_LIST": [ + { + "name": "Loopback0" + }, + { + "name": "Loopback0" + } + ], + "LOOPBACK_INTERFACE_IPPREFIX_LIST": [ + { + "family": "IPv6", + "ip-prefix": "aa::01/64", + "name": "Loopback0", + "scope": "global" + }, + { + "family": "IPv4", + "ip-prefix": "10.1.0.32/32", + "name": "Loopback0", + "scope": "global" + } + ] + } + }, + "sonic-acl:sonic-acl": { + "sonic-acl:ACL_TABLE": { + "ACL_TABLE_LIST": [ + { + "ACL_TABLE_NAME": "DATAACL", + "policy_desc": "DATAACL", + "ports": [ + "PortChannel1001", + "PortChannel1002" + ], + "stage": "ingress", + "type": "L3" + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "policy_desc": "EVERFLOW", + "ports": [ + "PortChannel1001", + "PortChannel1002" + ], + "stage": "ingress", + "type": "MIRROR" + }, + { + "ACL_TABLE_NAME": "EVERFLOWV6", + "policy_desc": "EVERFLOWV6", + "ports": [ + "PortChannel1001", + "PortChannel1002" + ], + "stage": "ingress", + "type": "MIRRORV6" + }, + { + "ACL_TABLE_NAME": "SNMP_ACL", + "policy_desc": "SNMP_ACL", + "services": [ + "SNMP" + ], + "stage": "ingress", + "type": "CTRLPLANE" + }, + { + "ACL_TABLE_NAME": "SSH_ONLY", + "policy_desc": "SSH_ONLY", + "services": [ + "SSH" + ], + "stage": "ingress", + "type": "CTRLPLANE" + } + ] + }, + "sonic-acl:ACL_RULE": { + "ACL_RULE_LIST": [ + { + "ACL_TABLE_NAME": "DATAACL", + "RULE_NAME": "Rule1", + "DST_IP": "192.168.1.1/24", + "SRC_IP": "10.10.1.1/16", + "IP_TYPE": "IPV4", + "PACKET_ACTION": "DROP", + "PRIORITY": 100 + + }, + { + "ACL_TABLE_NAME": "EVERFLOW", + "DST_IP": "192.169.10.1/32", + "SRC_IP": "10.10.1.1/16", + "IP_TYPE": "IPV4", + "RULE_NAME": "Rule2" + } + ] + } + }, + "sonic-vlan:sonic-vlan": { + "sonic-vlan:VLAN": { + "VLAN_LIST": [ + { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "192.169.1.1", + "198.169.1.1", + "199.169.1.2" + ], + "mtu": "9100", + "name": "Vlan100", + "vlanid": 100 + } + ] + }, + "sonic-vlan:VLAN_MEMBER": { + "VLAN_MEMBER_LIST": [ + { + "port": "Ethernet24", + "tagging_mode": "untagged", + "name": "Vlan100" + }, + { + "port": "Ethernet28", + "tagging_mode": "untagged", + "name": "Vlan100" + } + ] + }, + "sonic-vlan:VLAN_INTERFACE": { + "VLAN_INTERFACE_LIST": [ + { + "name": "Vlan100" + } + ], + "VLAN_INTERFACE_IPPREFIX_LIST": [ + { + "family": "IPv6", + "ip-prefix": "bb::01/64", + "scope": "global", + "name": "Vlan100" + }, + { + "family": "IPv4", + "ip-prefix": "10.35.61.1/24", + "scope": "global", + "name": "Vlan100" + } + ] + } + }, + "sonic-crm:sonic-crm": { + "sonic-crm:CRM": { + "Config": { + "acl_counter_high_threshold": 85, + "acl_counter_low_threshold": 25, + "acl_counter_threshold_type": "used", + "ipv4_route_high_threshold": 90, + "ipv4_route_low_threshold": 10, + "ipv4_route_threshold_type": "used", + "ipv6_route_high_threshold": 90, + "ipv6_route_low_threshold": 10, + "ipv6_route_threshold_type": "used", + "ipv4_neighbor_high_threshold": 85, + "ipv4_neighbor_low_threshold": 25, + "ipv4_neighbor_threshold_type": "used", + "ipv6_neighbor_high_threshold": 90, + "ipv6_neighbor_low_threshold": 10, + "ipv6_neighbor_threshold_type": "used" + } + } + } +} \ No newline at end of file diff --git a/src/sonic-yang-mgmt/sonic_yang_ext.py b/src/sonic-yang-mgmt/sonic_yang_ext.py index b01225277a..d6ad165742 100644 --- a/src/sonic-yang-mgmt/sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/sonic_yang_ext.py @@ -37,8 +37,8 @@ class SonicYangExtMixin: # keep only modules name in self.yangFiles self.yangFiles = [f.split('/')[-1] for f in self.yangFiles] self.yangFiles = [f.split('.')[0] for f in self.yangFiles] - print('Loaded below Yang Models') - print(self.yangFiles) + self.sysLog(syslog.LOG_DEBUG,'Loaded below Yang Models') + self.sysLog(syslog.LOG_DEBUG,self.yangFiles) # load json for each yang model self._loadJsonYangModel() @@ -578,8 +578,13 @@ class SonicYangExtMixin: for module_top in yangJ.keys(): # module _top will be of from module:top for container in yangJ[module_top].keys(): - #table = container.split(':')[1] - table = container + # the module_top can the format + # moduleName:TableName or + # TableName + names = container.split(':') + if len(names) > 2: + raise SonicYangException("Invalid Yang data file structure") + table = names[0] if len(names) == 1 else names[1] #print("revXlate " + table) cmap = self.confDbYangMap[table] cDbJson[table] = dict() @@ -780,4 +785,13 @@ class SonicYangExtMixin: return True + + def XlateYangToConfigDB(self, yang_data): + config_db_json = dict() + self.xlateJson = yang_data + self.revXlateJson = config_db_json + self._revXlateYangtoConfigDB(yang_data, config_db_json) + return config_db_json + + # End of class sonic_yang diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index 2de2a87708..0566809f57 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -302,7 +302,7 @@ class Test_SonicYang(object): jIn = json.loads(jIn) numTables = len(jIn) # load config and create Data tree - syc.loadData(jIn, debug=True) + syc.loadData(jIn) # check all tables are loaded and config related to all Yang Models is # loaded in Data tree. assert len(syc.jIn) == numTables @@ -329,12 +329,12 @@ class Test_SonicYang(object): jIn = json.loads(jIn) numTables = len(jIn) - syc.loadData(jIn, debug=True) + syc.loadData(jIn) # check all tables are loaded and no tables is without Yang Models assert len(syc.jIn) == numTables assert len(syc.tablesWithOutYang) == 0 - syc.getData(debug=True) + syc.getData() if syc.jIn and syc.jIn == syc.revXlateJson: print("Xlate and Rev Xlate Passed")