From 9e34003136730eac406d64c9995a92db6beef809 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 26 Oct 2020 13:48:50 -0700 Subject: [PATCH] [sonic-config-engine] Clean up dependencies, pin versions; install Python 3 package in Buster container (#5656) To clean up the image build procedure, and let setuptools/pip[3] implicitly install Python dependencies. Also use ipaddress package instead of ipaddr. --- .../docker-config-engine-buster/Dockerfile.j2 | 30 +++++---- .../Dockerfile.j2 | 11 +--- dockers/docker-config-engine/Dockerfile.j2 | 6 +- dockers/docker-orchagent/Dockerfile.j2 | 24 ++++--- .../docker-sonic-mgmt-framework/Dockerfile.j2 | 2 +- dockers/dockerfile-macros.j2 | 2 +- .../build_templates/sonic_debian_extension.j2 | 5 +- platform/vs/docker-sonic-vs/Dockerfile.j2 | 24 +++---- rules/docker-config-engine-buster.mk | 3 + rules/sonic-config.mk | 4 +- sonic-slave-buster/Dockerfile.j2 | 64 +++++++++++-------- sonic-slave-stretch/Dockerfile.j2 | 63 ++++++++++-------- src/sonic-bgpcfgd/setup.py | 44 ++++++++----- src/sonic-config-engine/minigraph.py | 24 ++++--- src/sonic-config-engine/setup.py | 50 ++++++++------- 15 files changed, 202 insertions(+), 154 deletions(-) diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2 index 93e1c98889..55156209f4 100644 --- a/dockers/docker-config-engine-buster/Dockerfile.j2 +++ b/dockers/docker-config-engine-buster/Dockerfile.j2 @@ -6,22 +6,25 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y \ - # Dependencies for sonic-cfggen - python-lxml \ - python-yaml \ - python-bitarray \ + build-essential \ python-pip \ + python3-pip \ python-dev \ + python3-dev \ apt-utils \ - python-setuptools + python-setuptools \ + python3-setuptools -RUN pip install --upgrade pip +RUN pip2 install --upgrade pip +RUN pip3 install --upgrade pip +RUN apt-get purge -y python-pip python3-pip -RUN pip install \ - netaddr \ - ipaddr \ - jinja2 \ - pyangbind==0.6.0 +# For sonic-config-engine Python 3 package +# Explicitly install pyangbind here, as pyangbind causes enum34 to be installed. +# enum34 causes Python 're' package to not work properly as it redefines an incompatible enum.py module +# https://github.com/robshakir/pyangbind/issues/232 +RUN pip3 install pyangbind==0.8.1 +RUN pip3 uninstall -y enum34 {% if docker_config_engine_buster_debs.strip() %} # Copy locally-built Debian package dependencies @@ -44,8 +47,9 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] ## Clean up RUN apt-get purge -y \ - python-pip \ - python-dev && \ + python-dev \ + python3-dev \ + build-essential && \ apt-get clean -y && \ apt-get autoclean -y && \ apt-get autoremove -y && \ diff --git a/dockers/docker-config-engine-stretch/Dockerfile.j2 b/dockers/docker-config-engine-stretch/Dockerfile.j2 index 21c5e65b77..8bc847d6fb 100644 --- a/dockers/docker-config-engine-stretch/Dockerfile.j2 +++ b/dockers/docker-config-engine-stretch/Dockerfile.j2 @@ -7,21 +7,13 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y \ # Dependencies for sonic-cfggen - python-lxml \ - python-yaml \ - python-bitarray \ + build-essential \ python-pip \ python-dev \ python-setuptools RUN pip install --upgrade pip -RUN pip install \ - netaddr \ - ipaddr \ - jinja2 \ - pyangbind==0.5.10 - {% if docker_config_engine_stretch_debs.strip() %} # Copy locally-built Debian package dependencies {{ copy_files("debs/", docker_config_engine_stretch_debs.split(' '), "/debs/") }} @@ -43,6 +35,7 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] ## Clean up RUN apt-get purge -y \ + build-essential \ python-pip \ python-dev && \ apt-get clean -y && \ diff --git a/dockers/docker-config-engine/Dockerfile.j2 b/dockers/docker-config-engine/Dockerfile.j2 index 6e5a56dbdf..c5adf15cb7 100644 --- a/dockers/docker-config-engine/Dockerfile.j2 +++ b/dockers/docker-config-engine/Dockerfile.j2 @@ -6,11 +6,11 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update # Dependencies for sonic-cfggen -RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev +RUN apt-get install -y build-essential python-pip python-dev RUN pip install --upgrade pip -RUN pip install netaddr ipaddr jinja2 pyangbind==0.5.10 +RUN apt-get purge -y python-pip {% if docker_config_engine_debs.strip() %} COPY \ @@ -46,5 +46,5 @@ python-wheels/{{ whl }}{{' '}} COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] ## Clean up -RUN apt-get purge -y python-pip python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN apt-get purge -y build-essential python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y RUN rm -rf /debs /python-wheels diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2 index 097a582f63..5af76e3c60 100755 --- a/dockers/docker-orchagent/Dockerfile.j2 +++ b/dockers/docker-orchagent/Dockerfile.j2 @@ -4,7 +4,7 @@ FROM docker-config-engine-buster ARG docker_container_name RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf -## Make apt-get non-interactive +# Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ @@ -18,10 +18,14 @@ RUN apt-get update && \ libmnl0 \ bridge-utils \ conntrack \ - ndppd + ndppd \ + # Needed for installing netifaces Python package + build-essential \ + python-dev \ + python3-dev {% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %} -## Fix for gcc/python not found in arm docker +# Fix for gcc/python not found in arm docker RUN apt-get install -f -y python2.7 python2.7-dev RUN apt-get install -y gcc-6 {% endif %} @@ -53,10 +57,14 @@ RUN apt-get remove -y gcc-6 {{ install_debian_packages(docker_orchagent_debs.split(' ')) }} {%- endif %} -## Clean up -RUN apt-get clean -y && \ - apt-get autoclean -y && \ - apt-get autoremove -y && \ +# Clean up +RUN apt-get purge -y \ + build-essential \ + python-dev \ + python3-dev && \ + apt-get clean -y && \ + apt-get autoclean -y && \ + apt-get autoremove -y && \ rm -rf /debs COPY ["files/arp_update", "/usr/bin"] @@ -68,7 +76,7 @@ COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] COPY ["critical_processes", "/etc/supervisor/"] -## Copy all Jinja2 template files into the templates folder +# Copy all Jinja2 template files into the templates folder COPY ["*.j2", "/usr/share/sonic/templates/"] ENTRYPOINT ["/usr/bin/docker-init.sh"] diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index dd396994d9..24ae3643b3 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -7,7 +7,7 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y g++ python-dev + apt-get install -y g++ python-dev libxml2 RUN pip install connexion==1.1.15 \ setuptools==21.0.0 \ diff --git a/dockers/dockerfile-macros.j2 b/dockers/dockerfile-macros.j2 index 5952d59f56..9917cb17ab 100644 --- a/dockers/dockerfile-macros.j2 +++ b/dockers/dockerfile-macros.j2 @@ -6,7 +6,7 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return {%- endmacro %} {% macro install_python2_wheels(packages) -%} -RUN cd /python-wheels/ && pip install {{ packages | join(' ') }} +RUN cd /python-wheels/ && pip2 install {{ packages | join(' ') }} {%- endmacro %} {% macro install_python3_wheels(packages) -%} diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 70b8b08a99..8a8334b79c 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -93,10 +93,7 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \ # Install dependencies for SONiC config engine sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \ python-dev \ - python3-dev \ - python-lxml \ - python-yaml \ - python-bitarray + python3-dev # Install Python client for Redis sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==3.5.3" diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2 index 3c9208756d..f254951d31 100644 --- a/platform/vs/docker-sonic-vs/Dockerfile.j2 +++ b/platform/vs/docker-sonic-vs/Dockerfile.j2 @@ -61,20 +61,20 @@ RUN curl -o redis-server_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.co RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb || apt-get install -f RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb -RUN pip install setuptools +RUN pip2 install setuptools RUN pip3 install setuptools -RUN pip install py2_ipaddress -RUN pip install six -RUN pip install pyroute2==0.5.3 netifaces==0.10.7 -RUN pip install monotonic==1.5 -RUN pip install urllib3 -RUN pip install requests -RUN pip install crontab +RUN pip2 install py2_ipaddress +RUN pip2 install six +RUN pip2 install pyroute2==0.5.3 netifaces==0.10.7 +RUN pip2 install monotonic==1.5 +RUN pip2 install urllib3 +RUN pip2 install requests +RUN pip2 install crontab # Install dependencies for Dynamic Port Breakout -RUN pip install xmltodict==0.12.0 -RUN pip install jsondiff==1.2.0 -RUN pip install ijson==2.6.1 +RUN pip2 install xmltodict==0.12.0 +RUN pip2 install jsondiff==1.2.0 +RUN pip2 install ijson==2.6.1 {% if docker_sonic_vs_debs.strip() -%} # Copy locally-built Debian package dependencies @@ -109,7 +109,7 @@ COPY python-wheels/{{ whl }} python-wheels/ # install PKGs after copying all PKGs to avoid dependency failure # use py3 to find python3 package, which is forced by wheel as of now {%- for whl in docker_sonic_vs_whls.split(' ') %} -RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }} +RUN pip{% if 'py3' in whl %}3{% else %}2{% endif %} install python-wheels/{{ whl }} {%- endfor %} {% endif %} diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index f1c54f10b6..9a98647a07 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -4,8 +4,11 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz $(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2) +$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) +$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) +$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) diff --git a/rules/sonic-config.mk b/rules/sonic-config.mk index f7ff062864..996e311b5f 100644 --- a/rules/sonic-config.mk +++ b/rules/sonic-config.mk @@ -6,11 +6,13 @@ $(SONIC_CONFIG_ENGINE_PY2)_DEPENDS += $(SWSSSDK_PY2) $(SONIC_PY_COMMON_PY2) $(SONIC_CONFIG_ENGINE_PY2)_PYTHON_VERSION = 2 SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) - # SONIC_CONFIG_ENGINE_PY3 package SONIC_CONFIG_ENGINE_PY3 = sonic_config_engine-1.0-py3-none-any.whl $(SONIC_CONFIG_ENGINE_PY3)_SRC_PATH = $(SRC_PATH)/sonic-config-engine $(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SWSSSDK_PY3) $(SONIC_PY_COMMON_PY3) +# Synthetic dependency to avoid building the Python 2 and 3 packages +# simultaneously and any potential conflicts which may arise +$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY2) $(SONIC_CONFIG_ENGINE_PY3)_PYTHON_VERSION = 3 SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 6b6902153b..ce406cd707 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -57,10 +57,10 @@ RUN apt-get update && apt-get install -y \ libtinyxml2-dev \ python \ python-pip \ + python3-pip \ libncurses5-dev \ texinfo \ dh-autoreconf \ - python3-pip \ doxygen \ devscripts \ git-buildpackage \ @@ -245,13 +245,6 @@ RUN apt-get update && apt-get install -y \ python3-sphinx \ # For sonic config engine testing python-dev \ - python-lxml \ - python3-lxml \ - python-jinja2 \ - python-netaddr \ - python-ipaddr \ - python-yaml \ - python3-yaml \ # For lockfile procmail \ # For gtest @@ -305,7 +298,14 @@ RUN apt-get update && apt-get install -y \ # For SWI Tools python-m2crypto \ # For build dtb - device-tree-compiler + device-tree-compiler \ +# For sonic-mgmt-framework + autoconf \ + m4 \ + libxml2-utils \ + xsltproc \ + python-lxml \ + libexpat1-dev ## Config dpkg ## install the configuration file if it’s currently missing @@ -334,29 +334,33 @@ RUN export VERSION=1.14.2 \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc \ && rm go$VERSION.linux-*.tar.gz +RUN pip2 install --upgrade pip +RUN pip3 install --upgrade pip +RUN apt-get purge -y python-pip python3-pip + # For building Python packages -RUN pip install setuptools==40.8.0 +RUN pip2 install setuptools==40.8.0 RUN pip3 install setuptools==49.6.00 # For building sonic-utilities -RUN pip install fastentrypoints +RUN pip2 install fastentrypoints RUN pip3 install fastentrypoints # For running Python unit tests -RUN pip install pytest-runner==4.4 +RUN pip2 install pytest-runner==4.4 RUN pip3 install pytest-runner==5.2 -RUN pip install mockredispy==2.9.3 +RUN pip2 install mockredispy==2.9.3 RUN pip3 install mockredispy==2.9.3 # For Python 2 unit tests, we need 'mock'. The last version of 'mock' # which supports Python 2 is 3.0.5. In Python 3, 'mock' is part of 'unittest' # in the standard library -RUN pip install mock==3.0.5 +RUN pip2 install mock==3.0.5 # For p4 build -RUN pip install \ - ctypesgen==1.0.2 \ - crc16 +RUN pip2 install \ + ctypesgen==1.0.2 \ + crc16 # For sonic config engine testing # Install pyangbind here, outside sonic-config-engine dependencies, as pyangbind causes enum34 to be installed. @@ -366,31 +370,35 @@ RUN pip3 install pyangbind==0.8.1 RUN pip3 uninstall -y enum34 # For templating -RUN pip install j2cli==0.3.10 +RUN pip2 install j2cli==0.3.10 -RUN pip3 install "PyYAML>=5.1" +# For sonic-mgmt-framework +RUN pip2 install "PyYAML==5.3.1" +RUN pip3 install "PyYAML==5.3.1" +RUN pip2 install "lxml==4.6.1" +RUN pip3 install "lxml==4.6.1" # For sonic-platform-common testing RUN pip3 install redis # For supervisor build -RUN pip install meld3 +RUN apt-get install python-meld3 # For vs image build -RUN pip install pexpect==4.6.0 +RUN pip2 install pexpect==4.6.0 # For sonic-swss-common testing -RUN pip install Pympler==0.8 +RUN pip2 install Pympler==0.8 # For sonic_yang_mgmt build -RUN pip install ijson==2.6.1 +RUN pip2 install ijson==2.6.1 RUN pip3 install ijson==2.6.1 -RUN pip install jsondiff==1.2.0 -RUN pip install xmltodict==0.12.0 -RUN pip install pyang==2.1.1 +RUN pip2 install jsondiff==1.2.0 +RUN pip2 install xmltodict==0.12.0 +RUN pip2 install pyang==2.1.1 # For mgmt-framework build -RUN pip install mmh3 +RUN pip2 install mmh3 RUN apt-get install -y xsltproc # Install dependencies for isc-dhcp-relay build @@ -434,4 +442,4 @@ RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs\"" >> /etc/default/d RUN update-alternatives --set iptables /usr/sbin/iptables-legacy # Install swi tools -RUN python -m pip install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00 +RUN pip2 install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00 diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 00935b33f1..3f455bdd7b 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -57,10 +57,10 @@ RUN apt-get update && apt-get install -y \ libtinyxml2-dev \ python \ python-pip \ + python3-pip \ libncurses5-dev \ texinfo \ dh-autoreconf \ - python3-pip \ doxygen \ devscripts \ git-buildpackage \ @@ -244,13 +244,6 @@ RUN apt-get update && apt-get install -y \ python3-sphinx \ # For sonic config engine testing python-dev \ - python-lxml \ - python3-lxml \ - python-jinja2 \ - python-netaddr \ - python-ipaddr \ - python-yaml \ - python3-yaml \ # For lockfile procmail \ # For pam_tacplus build @@ -298,7 +291,14 @@ RUN apt-get update && apt-get install -y \ # For libyang swig \ # For SWI Tools - python-m2crypto + python-m2crypto \ +# For sonic-mgmt-framework + autoconf \ + m4 \ + libxml2-utils \ + xsltproc \ + python-lxml \ + libexpat1-dev ## Config dpkg ## install the configuration file if it’s currently missing @@ -330,13 +330,17 @@ RUN export VERSION=1.14.2 \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc \ && rm go$VERSION.linux-*.tar.gz +RUN pip2 install --upgrade pip +RUN pip3 install --upgrade pip +RUN apt-get purge -y python-pip python3-pip + # For p4 build -RUN pip install \ - ctypesgen==0.r125 \ - crc16 +RUN pip2 install \ + ctypesgen==0.r125 \ + crc16 # Note: Stick with Jinja2 2.x branch as the 3.x dropped support for Python 2.7 -RUN pip install --force-reinstall --upgrade "Jinja2<3.0.0" +RUN pip2 install --force-reinstall --upgrade "Jinja2<3.0.0" # For sonic config engine testing # Install pyangbind here, outside sonic-config-engine dependencies, as pyangbind causes enum34 to be installed. @@ -346,39 +350,44 @@ RUN pip3 install pyangbind==0.8.1 RUN pip3 uninstall -y enum34 # For templating -RUN pip install j2cli==0.3.10 +RUN pip2 install j2cli==0.3.10 # For sonic snmpagent mock testing RUN pip3 install mockredispy==2.9.3 -RUN pip3 install "PyYAML>=5.1" +# For sonic-mgmt-framework +RUN pip2 install "PyYAML==5.3.1" +RUN pip3 install "PyYAML==5.3.1" +RUN pip2 install "lxml==4.6.1" +RUN pip3 install "lxml==4.6.1" + # For sonic-platform-common testing RUN pip3 install redis # For supervisor build -RUN pip install meld3 mock +RUN pip2 install meld3 mock # For vs image build -RUN pip install pexpect==4.6.0 +RUN pip2 install pexpect==4.6.0 # For sonic-utilities build -RUN pip install mockredispy==2.9.3 -RUN pip install pytest-runner==4.4 -RUN pip install setuptools==40.8.0 +RUN pip2 install mockredispy==2.9.3 +RUN pip2 install pytest-runner==4.4 +RUN pip2 install setuptools==40.8.0 # For sonic-swss-common testing -RUN pip install Pympler==0.8 +RUN pip2 install Pympler==0.8 # For sonic_yang_mgmt build -RUN pip install ijson==2.6.1 +RUN pip2 install ijson==2.6.1 RUN pip3 install ijson==2.6.1 -RUN pip install jsondiff==1.2.0 -RUN pip install xmltodict==0.12.0 -RUN pip install pyang==2.1.1 +RUN pip2 install jsondiff==1.2.0 +RUN pip2 install xmltodict==0.12.0 +RUN pip2 install pyang==2.1.1 # For mgmt-framework build -RUN pip install mmh3 +RUN pip2 install mmh3 # Install dependencies for isc-dhcp-relay build RUN apt-get -y build-dep isc-dhcp @@ -437,4 +446,4 @@ RUN apt-get install -y docker-ce=18.06.3~ce~3-0~debian RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs\"" >> /etc/default/docker # Install swi tools -RUN python -m pip install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00 +RUN pip2 install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00 diff --git a/src/sonic-bgpcfgd/setup.py b/src/sonic-bgpcfgd/setup.py index 1e23802d98..5b18bb2ba2 100755 --- a/src/sonic-bgpcfgd/setup.py +++ b/src/sonic-bgpcfgd/setup.py @@ -2,20 +2,32 @@ import setuptools -setuptools.setup(name='sonic-bgpcfgd', - version='1.0', - description='Utility to dynamically generate BGP configuration for FRR', - author='Pavel Shirshov', - author_email='pavelsh@microsoft.com', - url='https://github.com/Azure/sonic-buildimage', - packages=setuptools.find_packages(), - entry_points={ - 'console_scripts': [ - 'bgpcfgd = bgpcfgd.main:main', - 'bgpmon = bgpmon.bgpmon:main', - ] - }, - install_requires=['jinja2>=2.10', 'netaddr', 'pyyaml'], - setup_requires=['pytest-runner'], - tests_require=['pytest', 'pytest-cov'], +setuptools.setup( + name = 'sonic-bgpcfgd', + version = '1.0', + description = 'Utility to dynamically generate BGP configuration for FRR', + author = 'Pavel Shirshov', + author_email = 'pavelsh@microsoft.com', + url = 'https://github.com/Azure/sonic-buildimage', + packages = setuptools.find_packages(), + entry_points = { + 'console_scripts': [ + 'bgpcfgd = bgpcfgd.main:main', + 'bgpmon = bgpmon.bgpmon:main', + ] + }, + install_requires = [ + 'jinja2>=2.10', + 'netaddr==0.8.0', + 'pyyaml==5.3.1', + 'zipp==1.2.0', # importlib-resources needs zipp and seems to have a bug where it will try to import too new of a version for Python 2 + ], + setup_requires = [ + 'pytest-runner', + 'wheel' + ], + tests_require = [ + 'pytest', + 'pytest-cov' + ] ) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 294fb7faaa..c4e8a6ae21 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -8,7 +8,7 @@ import socket import struct import json import copy -import ipaddr as ipaddress +import ipaddress from collections import defaultdict from lxml import etree as ET @@ -18,6 +18,12 @@ from portconfig import get_port_config from sonic_py_common.multi_asic import get_asic_id_from_name from sonic_py_common.interface import backplane_prefix +# TODO: Remove this once we no longer support Python 2 +if sys.version_info.major == 3: + UNICODE_TYPE = str +else: + UNICODE_TYPE = unicode + """minigraph.py version_added: "1.9" author: Guohan Lu (gulv@microsoft.com) @@ -323,8 +329,8 @@ def parse_dpg(dpg, hname): for mgmtintf in mgmtintfs.findall(str(QName(ns1, "ManagementIPInterface"))): intfname = mgmtintf.find(str(QName(ns, "AttachTo"))).text ipprefix = mgmtintf.find(str(QName(ns1, "PrefixStr"))).text - mgmtipn = ipaddress.IPNetwork(ipprefix) - gwaddr = ipaddress.IPAddress(int(mgmtipn.network) + 1) + mgmtipn = ipaddress.ip_network(UNICODE_TYPE(ipprefix), False) + gwaddr = ipaddress.ip_address(next(mgmtipn.hosts())) mgmt_intf[(intfname, ipprefix)] = {'gwaddr': gwaddr} pcintfs = child.find(str(QName(ns, "PortChannelInterfaces"))) @@ -665,9 +671,9 @@ def parse_spine_chassis_fe(results, vni, lo_intfs, phyport_intfs, pc_intfs, pc_m # Vxlan tunnel information lo_addr = '0.0.0.0' for lo in lo_intfs: - lo_network = ipaddress.IPNetwork(lo[1]) + lo_network = ipaddress.ip_network(UNICODE_TYPE(lo[1]), False) if lo_network.version == 4: - lo_addr = str(lo_network.ip) + lo_addr = str(lo_network.network_address) break results['VXLAN_TUNNEL'] = {chassis_vxlan_tunnel: { 'src_ip': lo_addr @@ -1174,9 +1180,9 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw # if erspan_dst: # lo_addr = '0.0.0.0' # for lo in lo_intfs: - # lo_network = ipaddress.IPNetwork(lo[1]) + # lo_network = ipaddress.ip_network(UNICODE_TYPE(lo[1]), False) # if lo_network.version == 4: - # lo_addr = str(lo_network.ip) + # lo_addr = str(lo_network.network_address) # break # count = 0 # for dst in erspan_dst: @@ -1204,8 +1210,8 @@ def parse_device_desc_xml(filename): results['LOOPBACK_INTERFACE'] = {('lo', lo_prefix): {}} mgmt_intf = {} - mgmtipn = ipaddress.IPNetwork(mgmt_prefix) - gwaddr = ipaddress.IPAddress(int(mgmtipn.network) + 1) + mgmtipn = ipaddress.ip_network(UNICODE_TYPE(mgmt_prefix), False) + gwaddr = ipaddress.ip_address((next(mgmtipn.hosts()))) results['MGMT_INTERFACE'] = {('eth0', mgmt_prefix): {'gwaddr': gwaddr}} return results diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index e1940d4c06..7e7b9b4551 100755 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -1,32 +1,38 @@ import glob +import sys from setuptools import setup -from tests.common_utils import PY3x - +# Common dependencies for Python 2 and 3 dependencies = [ -# Python 2 or 3 dependencies + 'bitarray==1.5.3', + 'ipaddress==1.0.23', + 'lxml==4.6.1', + 'netaddr==0.8.0', + 'pyyaml==5.3.1', + 'sonic-py-common', +] + +if sys.version_info.major == 3: + # Python 3-only dependencies + dependencies += [ + # pyangbind v0.8.1 pull down enum43 which causes 're' package to malfunction. + # Python3 has enum module and so pyangbind should be installed outside + # dependencies section of setuptools followed by uninstall of enum43 + # 'pyangbind==0.8.1', + 'Jinja2>=2.10' + ] +else: + # Python 2-only dependencies + dependencies += [ + # Jinja2 v3.0.0+ dropped support for Python 2.7 and causes setuptools to + # malfunction on stretch slave docker. 'future', - 'ipaddr', - 'lxml', - 'netaddr', - 'pyyaml', - 'sonic-py-common', - ] + ([ -# Python 3 dependencies -# pyangbind v0.8.1 pull down enum43 which causes 're' package to malfunction. -# Python3 has enum module and so pyangbind should be installed outside -# dependencies section of setuptools followed by uninstall of enum43 -# 'pyangbind==0.8.1', - 'Jinja2>=2.10', - ] if PY3x - else [ -# Python 2 dependencies -# Jinja2 v3.0.0+ dropped support for Python 2.7 and causes setuptools to -# malfunction on stretch slave docker. 'Jinja2<3.0.0', 'pyangbind==0.6.0', - ]) + 'zipp==1.2.0', # importlib-resources needs zipp and seems to have a bug where it will try to install too new of a version for Python 2 + ] + setup( name = 'sonic-config-engine', @@ -52,6 +58,7 @@ setup( ], setup_requires= [ 'pytest-runner', + 'wheel' ], tests_require=[ 'pytest', @@ -69,4 +76,3 @@ setup( ], keywords = 'SONiC sonic-cfggen config-engine PYTHON python' ) -