[sonic-yang-mgmt] Build PY3 & PY2 packages (#5559)

Moving sonic-yang-mgmt to PY3 to support move of sonic-utilities to PY3.

Signed-off-by: Praveen Chaudhary<pchaudhary@linkedin.com>
This commit is contained in:
Praveen Chaudhary 2020-11-07 13:03:41 -08:00 committed by GitHub
parent 04d0e8ab00
commit 6156cb2805
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 145 additions and 143 deletions

View File

@ -154,15 +154,23 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY2_WHEEL_NAME
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb
SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}})
sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
# Install sonic-yang-mgmt Python package
SONIC_YANG_MGMT_PY_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py_wheel_path}})
sudo cp {{sonic_yang_mgmt_py_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install $SONIC_YANG_MGMT_PY_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
# Install sonic-yang-mgmt Python2 package
SONIC_YANG_MGMT_PY2_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py2_wheel_path}})
sudo cp {{sonic_yang_mgmt_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install $SONIC_YANG_MGMT_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
# Install sonic-yang-mgmt Python3 package
SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}})
sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
# Install sonic-platform-common Python 2 package
PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})

View File

@ -22,7 +22,7 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
$(SONIC_PY_COMMON_PY2) \
$(SONIC_PY_COMMON_PY3) \
$(SONIC_YANG_MODELS_PY3) \
$(SONIC_YANG_MGMT_PY) \
$(SONIC_YANG_MGMT_PY2) \
$(SONIC_UTILITIES_PY2) \
$(SONIC_HOST_SERVICES_PY3)

View File

@ -83,11 +83,6 @@ RUN pip2 install urllib3
RUN pip2 install requests
RUN pip2 install crontab
# Install dependencies for Dynamic Port Breakout
RUN pip2 install xmltodict==0.12.0
RUN pip2 install jsondiff==1.2.0
RUN pip2 install ijson==2.6.1
{% if docker_sonic_vs_debs.strip() -%}
# Copy locally-built Debian package dependencies
{%- for deb in docker_sonic_vs_debs.split(' ') %}

View File

@ -14,7 +14,7 @@ $(SONIC_UTILITIES_PY2)_DEPENDS += $(SONIC_PY_COMMON_PY2) \
$(SONIC_PY_COMMON_PY3) \
$(SWSSSDK_PY2) \
$(SONIC_CONFIG_ENGINE_PY2) \
$(SONIC_YANG_MGMT_PY) \
$(SONIC_YANG_MGMT_PY2) \
$(SONIC_YANG_MODELS_PY3)
$(SONIC_UTILITIES_PY2)_DEBS_DEPENDS = $(LIBYANG) \
$(LIBYANG_CPP) \

View File

@ -1,10 +1,10 @@
SPATH := $($(SONIC_YANG_MGMT_PY)_SRC_PATH)
SPATH := $($(SONIC_YANG_MGMT_PY2)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-yang-mgmt-py2.mk rules/sonic-yang-mgmt-py2.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(SPATH))
$(SONIC_YANG_MGMT_PY)_CACHE_MODE := GIT_CONTENT_SHA
$(SONIC_YANG_MGMT_PY)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(SONIC_YANG_MGMT_PY)_DEP_FILES := $(DEP_FILES)
$(SONIC_YANG_MGMT_PY2)_CACHE_MODE := GIT_CONTENT_SHA
$(SONIC_YANG_MGMT_PY2)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(SONIC_YANG_MGMT_PY2)_DEP_FILES := $(DEP_FILES)

View File

@ -1,12 +1,12 @@
# sonic-yang-mgmt python2 wheel
SONIC_YANG_MGMT_PY = sonic_yang_mgmt-1.0-py2-none-any.whl
$(SONIC_YANG_MGMT_PY)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
$(SONIC_YANG_MGMT_PY)_PYTHON_VERSION = 2
$(SONIC_YANG_MGMT_PY)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY2) \
SONIC_YANG_MGMT_PY2 = sonic_yang_mgmt-1.0-py2-none-any.whl
$(SONIC_YANG_MGMT_PY2)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
$(SONIC_YANG_MGMT_PY2)_PYTHON_VERSION = 2
$(SONIC_YANG_MGMT_PY2)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY2) \
$(LIBYANG_PY3)
$(SONIC_YANG_MGMT_PY)_DEPENDS = $(SONIC_YANG_MODELS_PY3)
$(SONIC_YANG_MGMT_PY)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
$(SONIC_YANG_MGMT_PY2)_DEPENDS = $(SONIC_YANG_MODELS_PY3)
$(SONIC_YANG_MGMT_PY2)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
$(LIBYANG_CPP) $(LIBYANG_PY2)
SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY)
SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY2)

View File

@ -0,0 +1,10 @@
SPATH := $($(SONIC_YANG_MGMT_PY3)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-yang-mgmt-py3.mk rules/sonic-yang-mgmt-py3.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(SPATH))
$(SONIC_YANG_MGMT_PY3)_CACHE_MODE := GIT_CONTENT_SHA
$(SONIC_YANG_MGMT_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(SONIC_YANG_MGMT_PY3)_DEP_FILES := $(DEP_FILES)

View File

@ -0,0 +1,11 @@
# sonic-yang-mgmt python3 wheel
SONIC_YANG_MGMT_PY3 = sonic_yang_mgmt-1.0-py3-none-any.whl
$(SONIC_YANG_MGMT_PY3)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
$(SONIC_YANG_MGMT_PY3)_PYTHON_VERSION = 3
$(SONIC_YANG_MGMT_PY3)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY3)
$(SONIC_YANG_MGMT_PY3)_DEPENDS = $(SONIC_YANG_MODELS_PY3)
$(SONIC_YANG_MGMT_PY3)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
$(LIBYANG_CPP) $(LIBYANG_PY3)
SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY3)

View File

@ -821,7 +821,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
$(HEADER)
@ -864,7 +865,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))"
export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"
export sonic_yang_mgmt_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))"
export sonic_yang_mgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3))"
export multi_instance="false"
export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"

View File

@ -390,11 +390,7 @@ RUN pip2 install pexpect==4.6.0
# For sonic-swss-common testing
RUN pip2 install Pympler==0.8
# For sonic_yang_mgmt build
RUN pip2 install ijson==2.6.1
RUN pip3 install ijson==2.6.1
RUN pip2 install jsondiff==1.2.0
RUN pip2 install xmltodict==0.12.0
# For sonic_yang_model build
RUN pip2 install pyang==2.1.1
# For mgmt-framework build

View File

@ -339,7 +339,7 @@ RUN pip2 install \
ctypesgen==0.r125 \
crc16
# Note: Stick with Jinja2 2.x branch as the 3.x dropped support for Python 2.7
# Note: Stick with Jinja2 2.x branch as the 3.x dropped support for Python 2.7
RUN pip2 install --force-reinstall --upgrade "Jinja2<3.0.0"
# For sonic config engine testing
@ -379,11 +379,7 @@ RUN pip2 install setuptools==40.8.0
# For sonic-swss-common testing
RUN pip2 install Pympler==0.8
# For sonic_yang_mgmt build
RUN pip2 install ijson==2.6.1
RUN pip3 install ijson==2.6.1
RUN pip2 install jsondiff==1.2.0
RUN pip2 install xmltodict==0.12.0
# For sonic_yang_model build
RUN pip2 install pyang==2.1.1
# For mgmt-framework build

View File

@ -0,0 +1,2 @@
[aliases]
test=pytest

View File

@ -4,37 +4,12 @@
"""The setup script."""
from setuptools import setup, find_packages
from setuptools.command.build_py import build_py
from os import system
from sys import exit
import pytest
setup_requirements = ['pytest-runner']
test_requirements = ['pytest>=3']
# read me
with open('README.rst') as readme_file:
readme = readme_file.read()
# class for prerequisites to build this package
class pkgBuild(build_py):
"""Custom Build PLY"""
def run (self):
# run pytest for libyang python APIs
self.pytest_args = []
errno = pytest.main(self.pytest_args)
if (errno):
exit(errno)
# Continue usual build steps
build_py.run(self)
setup(
cmdclass={
'build_py': pkgBuild,
},
author="lnos-coders",
author_email='lnos-coders@linkedin.com',
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
@ -52,15 +27,26 @@ setup(
'Programming Language :: Python :: 3.8',
],
description="Package contains Python Library for YANG for sonic.",
tests_require = test_requirements,
license="GNU General Public License v3",
long_description=readme + '\n\n',
install_requires = [
'xmltodict==0.12.0',
'ijson==2.6.1'
],
tests_require = [
'pytest>3',
'xmltodict==0.12.0',
'ijson==2.6.1'
],
setup_requires = [
'pytest-runner',
'wheel'
],
include_package_data=True,
keywords='sonic_yang_mgmt',
name='sonic_yang_mgmt',
keywords='sonic-yang-mgmt',
name='sonic-yang-mgmt',
py_modules=['sonic_yang', 'sonic_yang_ext'],
packages=find_packages(),
setup_requires=setup_requirements,
version='1.0',
zip_safe=False,
)

View File

@ -129,7 +129,8 @@ class SonicYangExtMixin:
"""
def _cropConfigDB(self, croppedFile=None):
for table in self.jIn.keys():
tables = list(self.jIn.keys())
for table in tables:
if table not in self.confDbYangMap:
# store in tablesWithOutYang
self.tablesWithOutYang[table] = self.jIn[table]
@ -138,7 +139,7 @@ class SonicYangExtMixin:
if len(self.tablesWithOutYang):
print("Note: Below table(s) have no YANG models:")
for table in self.tablesWithOutYang.keys():
print(unicode(table), end=", ")
print(str(table), end=", ")
print()
if croppedFile:
@ -274,13 +275,14 @@ class SonicYangExtMixin:
# fetch regex from YANG models.
keyRegEx = model['ext:key-regex-configdb-to-yang']['@value']
# seperator `|` has special meaning in regex, so change it appropriately.
keyRegEx = re.sub('\|', '\\|', keyRegEx)
keyRegEx = re.sub(r'\|', r'\\|', keyRegEx)
# get keys from YANG model list itself
listKeys = model['key']['@value']
self.sysLog(msg="xlateList regex:{} keyList:{}".\
format(keyRegEx, listKeys))
for pkey in config.keys():
primaryKeys = list(config.keys())
for pkey in primaryKeys:
try:
vKey = None
self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\

View File

@ -64,8 +64,8 @@ class Test_SonicYang(object):
def load_yang_model_file(self, yang_s, yang_dir, yang_file, module_name):
yfile = yang_dir + yang_file
try:
yang_s._load_schema_module(str(yfile))
except Exception as e:
yang_s._load_schema_module(str(yfile))
except Exception as e:
print(e)
raise
@ -106,7 +106,7 @@ class Test_SonicYang(object):
data_files = []
data_files.append(data_file)
data_files.append(data_merge_file)
print(yang_files)
print(yang_files)
yang_s._load_data_model(yang_dir, yang_files, data_files)
#validate the data tree from data_merge_file is loaded

View File

@ -0,0 +1,2 @@
[aliases]
test=pytest

View File

@ -4,51 +4,12 @@
"""The setup script."""
from setuptools import setup, find_packages
from setuptools.command.build_py import build_py
from os import system
from sys import exit
setup_requirements = ['pytest-runner']
test_requirements = ['pytest>=3']
# read me
with open('README.rst') as readme_file:
readme = readme_file.read()
# class for prerequisites to build this package
class pkgBuild(build_py):
"""Custom Build PLY"""
def run (self):
# json file for YANG model test cases.
test_yangJson_file = './tests/yang_model_tests/yangTest.json'
# YANG models are in below dir
yang_model_dir = './yang-models/'
# yang model tester python module
yang_test_py = './tests/yang_model_tests/yangModelTesting.py'
# run tests for yang models
test_yang_cmd = "python {} -f {} -y {}".format(yang_test_py, test_yangJson_file, yang_model_dir)
if (system(test_yang_cmd)):
print("YANG Tests failed\n")
exit(1)
else:
print("YANG Tests passed\n")
# Generate YANG Tree
pyang_tree_cmd = "pyang -f tree ./yang-models/*.yang > ./yang-models/sonic_yang_tree"
if (system(pyang_tree_cmd)):
print("Failed: {}".format(pyang_tree_cmd))
else:
print("Passed: {}".format(pyang_tree_cmd))
# Continue usual build steps
build_py.run(self)
setup(
cmdclass={
'build_py': pkgBuild,
},
author="lnos-coders",
author_email='lnos-coders@linkedin.com',
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
@ -66,15 +27,23 @@ setup(
'Programming Language :: Python :: 3.8',
],
description="Package contains YANG models for sonic.",
tests_require = test_requirements,
license="GNU General Public License v3",
long_description=readme + '\n\n',
install_requires = [
],
tests_require = [
'pytest',
'ijson==2.6.1'
],
setup_requires = [
'pytest-runner',
'wheel'
],
include_package_data=True,
keywords='sonic_yang_models',
name='sonic_yang_models',
keywords='sonic-yang-models',
name='sonic-yang-models',
py_modules=[],
packages=find_packages(),
setup_requires=setup_requirements,
version='1.0',
data_files=[
('yang-models', ['./yang-models/sonic-types.yang',

View File

@ -5,6 +5,9 @@
import pytest
from os import system
from sys import exit
@pytest.fixture
def response():
"""Sample pytest fixture.
@ -14,8 +17,21 @@ def response():
# import requests
# return requests.get('https://github.com/audreyr/cookiecutter-pypackage')
def test_content(response):
"""Sample pytest test function with the pytest fixture as an argument."""
def test_generate_yang_tree():
# Generate YANG Tree, see no error in it.
pyang_tree_cmd = "pyang -f tree ./yang-models/*.yang > ./yang-models/sonic_yang_tree"
if (system(pyang_tree_cmd)):
print("Failed: {}".format(pyang_tree_cmd))
exit(1)
else:
print("Passed: {}".format(pyang_tree_cmd))
return
# from bs4 import BeautifulSoup
# assert 'GitHub' in BeautifulSoup(response.content).title.string

View File

@ -34,9 +34,9 @@ def printExceptionDetails():
# Run function will run all the tests
# from a user given list.
class YangModelTesting:
class Test_yang_models:
def __init__(self, tests, yangDir, jsonFile):
def initTest(self):
self.defaultYANGFailure = {
'Must': ['Must condition', 'not satisfied'],
'InvalidValue': ['Invalid value'],
@ -178,11 +178,9 @@ class YangModelTesting:
}
}
self.tests = tests
if (self.tests == None):
self.tests = self.ExceptionTests.keys()+self.SpecialTests.keys()
self.yangDir = yangDir
self.jsonFile = jsonFile
self.tests = list(self.ExceptionTests.keys())+list(self.SpecialTests.keys())
self.yangDir = './yang-models/'
self.jsonFile = './tests/yang_model_tests/yangTest.json'
self.testNum = 1
# other class vars
# self.ctx
@ -212,24 +210,6 @@ class YangModelTesting:
raise e
return
"""
Run all tests from list self.tests
"""
def run(self):
try:
self.loadYangModel(self.yangDir)
ret = 0
for test in self.tests:
test = test.strip()
if test in self.ExceptionTests:
ret = ret + self.runExceptionTest(test);
elif test in self.SpecialTests:
ret = ret + self.runSpecialTest(test);
except Exception as e:
printExceptionDetails()
raise e
return ret
"""
Get the JSON input based on func name
and return jsonInput
@ -313,6 +293,8 @@ class YangModelTesting:
elif (sum(1 for str in eStr if str not in s) == 0):
log.info(desc + " Passed\n")
return PASS
else:
raise Exception("Unknown Error")
except Exception as e:
printExceptionDetails()
log.info(desc + " Failed\n")
@ -336,7 +318,7 @@ class YangModelTesting:
self.logStartTest(desc)
jInput = json.loads(self.readJsonInput(test))
# check all Vlan from 1 to 4094
for i in xrange(4095):
for i in range(4095):
vlan = 'Vlan'+str(i)
jInput["sonic-vlan:sonic-vlan"]["sonic-vlan:VLAN"]["VLAN_LIST"]\
[0]["vlan_name"] = vlan
@ -349,6 +331,27 @@ class YangModelTesting:
printExceptionDetails()
log.info(desc + " Failed\n")
return FAIL
"""
Run all tests from list self.tests
"""
def test_run_tests(self):
try:
self.initTest()
self.loadYangModel(self.yangDir)
ret = 0
for test in self.tests:
test = test.strip()
if test in self.ExceptionTests:
ret = ret + self.runExceptionTest(test);
elif test in self.SpecialTests:
ret = ret + self.runSpecialTest(test);
else:
raise Exception("Unexpected Test")
except Exception as e:
printExceptionDetails()
assert ret == 0
return
# End of Class
"""

View File

@ -188,7 +188,7 @@
"sonic-vlan:sonic-vlan": {
"sonic-vlan:VLAN_MEMBER": {
"VLAN_MEMBER_LIST": [{
"vlan_name": 100,
"vlan_name": "Vlan100",
"port": "Ethernet0",
"tagging_mode": "non-tagged"
}]

View File

@ -1,5 +1,7 @@
module sonic-loopback-interface {
yang-version 1.1;
namespace "http://github.com/Azure/sonic-loopback-interface";
prefix lointf;

View File

@ -1,5 +1,7 @@
module sonic-vlan {
yang-version 1.1;
namespace "http://github.com/Azure/sonic-vlan";
prefix vlan;