[python-click] No longer build our own package, let pip/setuptools install vanilla (#5549)

We were building our own python-click package because we needed features/bug fixes available as of version 7.0.0, but the most recent version available from Debian was in the 6.x range.

"Click" is needed for building/testing and installing sonic-utilities. Now that we are building sonic-utilities as a wheel, with Click specified as a dependency in the setup.py file, setuptools will install a more recent version of Click in the sonic-slave-buster container when building the package, and pip will install a more recent version of Click in the host OS of SONiC when installing the sonic-utilities package. Also, we don't need to worry about installing the Python 2 or 3 version of the package, as the proper one will be installed as necessary.

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -12,7 +12,6 @@ RUN apt-get update         && \
         python-bitarray       \
         python-pip            \
         python-dev            \
-        python-natsort        \
         apt-utils             \
         python-setuptools
 

dockers/docker-config-engine-stretch/Dockerfile.j2

@@ -12,7 +12,6 @@ RUN apt-get update         && \
         python-bitarray       \
         python-pip            \
         python-dev            \
-        python-natsort        \
         python-setuptools
 
 RUN pip install --upgrade pip

dockers/docker-config-engine/Dockerfile.j2

@@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 
 # Dependencies for sonic-cfggen
-RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev python-natsort
+RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev
 
 RUN pip install --upgrade pip
 

files/build_templates/sonic_debian_extension.j2

@@ -93,6 +93,7 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \
 # Install dependencies for SONiC config engine
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
     python-dev     \
+    python3-dev    \
     python-lxml    \
     python-yaml    \
     python-bitarray
@@ -184,14 +185,12 @@ sudo cp {{platform_common_py3_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY3
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $PLATFORM_COMMON_PY3_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY3_WHEEL_NAME
 
-# Install Debian packages and their dependencies which are needed by sonic-utilities
+# Install prerequisites needed for installing the Python m2crypto package, used by sonic-utilities
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f python-m2crypto
+# These packages can be uninstalled after intallation
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install build-essential libssl-dev swig
 
-# Install built Python Click package (and its dependencies via 'apt-get -y install -f')
+# Install prerequisites needed for using the Python m2crypto package, used by sonic-utilities
-# Do this before installing sonic-utilities so that it doesn't attempt to install
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install openssl
-# an older version as part of its dependencies
-sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python-click*_all.deb || \
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
 # Install SONiC Utilities Python 2 package
 SONIC_UTILITIES_PY2_WHEEL_NAME=$(basename {{sonic_utilities_py2_wheel_path}})
@@ -574,7 +573,8 @@ sudo cp $BUILD_TEMPLATES/mgmt-framework.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_S
 echo "mgmt-framework.timer" | sudo tee -a $GENERATED_SERVICE_FILE
 {% endif %}
 
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev python3-dev
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y build-essential libssl-dev swig
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
 

platform/p4/docker-sonic-p4/Dockerfile.j2

@@ -40,12 +40,18 @@ RUN apt-get install -y net-tools \
                        iproute \
                        libpython2.7 \
                        grub2-common \
-                       python-click \
-                       python-natsort \
-                       python-tabulate \
                        bash-completion \
                        libelf1 \
-                       libmnl0
+                       libmnl0 \
+                       # For installing Python m2crypto package
+                       # (these can be uninstalled after installation)
+                       build-essential \
+                       python-dev \
+                       python3-dev \
+                       libssl-dev \
+                       swig \
+                       # For using Python m2crypto package
+                       openssl
 
 RUN pip install setuptools
 RUN pip install py2_ipaddress
@@ -62,6 +68,8 @@ debs/{{ deb }}{{' '}}
 {%- endfor %}
 
 ## Clean up
+RUN apt-get purge -y build-essential libssl-dev swig
+RUN apt-get purge -y python-dev python3-dev
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs
 

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -32,9 +32,6 @@ RUN apt-get install -y net-tools \
                        libc-ares2 \
                        iproute2 \
                        grub2-common \
-                       python-click \
-                       python-natsort \
-                       python-tabulate \
                        bash-completion \
                        libelf1 \
                        libmnl0 \
@@ -47,8 +44,16 @@ RUN apt-get install -y net-tools \
                        iptables \
                        python3-pip \
                        jq \
-                       python-m2crypto \
+                       libzmq5 \
-                       libzmq5
+                       # For installing Python m2crypto package
+                       # (these can be uninstalled after installation)
+                       build-essential \
+                       python-dev \
+                       python3-dev \
+                       libssl-dev \
+                       swig \
+                       # For using Python m2crypto package
+                       openssl
 
 # install redis-server
 RUN curl -o redis-tools_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.core.windows.net/packages/redis/redis-tools_6.0.6-1~bpo10+1_amd64.deb?sv=2015-04-05&sr=b&sig=73zbmjkf3pi%2Bn0R8Hy7CWT2EUvOAyzM5aLYJWCLySGM%3D&se=2030-09-06T19%3A44%3A59Z&sp=r"
@@ -111,6 +116,8 @@ RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
 {% endif %}
 
 # Clean up
+RUN apt-get purge -y build-essential libssl-dev swig
+RUN apt-get purge -y python-dev python3-dev
 RUN apt-get clean -y
 RUN apt-get autoclean -y
 RUN apt-get autoremove -y

rules/python-click.dep

@@ -1,10 +0,0 @@
-
-SPATH       := $($(PYTHON_CLICK)_SRC_PATH)
-DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/python-click.mk rules/python-click.dep   
-DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
-DEP_FILES   += $(shell git ls-files $(SPATH))
-
-$(PYTHON_CLICK)_CACHE_MODE  := GIT_CONTENT_SHA 
-$(PYTHON_CLICK)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
-$(PYTHON_CLICK)_DEP_FILES   := $(DEP_FILES)
-

rules/python-click.mk

@@ -1,16 +0,0 @@
-# python-click package
-#
-# Python Click versions < 6.7 have a bug which causes bash completion
-# functionality to stop working after two sublevels. sonic-utilities depends
-# on this package, and the most recent version provided by Debian Jessie and
-# Stretch is v6.6. We build version 6.7 from source in order to fix this bug.
-# TODO: If we upgrade to a distro which provides a version >= 6.7 we will no
-# longer need to build this.
-
-PYTHON_CLICK_VERSION = 7.0-1
-
-export PYTHON_CLICK_VERSION
-
-PYTHON_CLICK = python-click_$(PYTHON_CLICK_VERSION)_all.deb
-$(PYTHON_CLICK)_SRC_PATH = $(SRC_PATH)/python-click
-SONIC_MAKE_DEBS += $(PYTHON_CLICK)

sonic-slave-buster/Dockerfile.j2

@@ -270,17 +270,6 @@ RUN apt-get update && apt-get install -y \
         texlive-latex-extra \
         texlive-latex-recommended \
         iproute2 \
-# For python-click build
-        python-sphinx \
-        python3-pallets-sphinx-themes \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -349,6 +338,10 @@ RUN export VERSION=1.14.2 \
 RUN pip install setuptools==40.8.0
 RUN pip3 install setuptools==49.6.00
 
+# For building sonic-utilities
+RUN pip install fastentrypoints
+RUN pip3 install fastentrypoints
+
 # For running Python unit tests
 RUN pip install pytest-runner==4.4
 RUN pip3 install pytest-runner==5.2
@@ -375,11 +368,6 @@ RUN pip3 uninstall -y enum34
 # For templating
 RUN pip install j2cli==0.3.10
 
-# Remove python-click 6.6
-RUN apt-get purge -y python-click
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 RUN pip3 install "PyYAML>=5.1"
 
 # For sonic-platform-common testing

sonic-slave-jessie/Dockerfile.j2

@@ -246,16 +246,6 @@ RUN apt-get update && apt-get install -y \
         lynx \
         texlive-latex-extra \
         texlive-latex-recommended \
-# For python-click build
-        python-sphinx \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -310,9 +300,6 @@ RUN pip install --force-reinstall --upgrade "jinja2>=2.10"
 # For templating (requiring jinja2)
 RUN pip install j2cli==0.3.10
 
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 # For supervisor build
 RUN pip install meld3 mock
 

sonic-slave-stretch/Dockerfile.j2

@@ -266,16 +266,6 @@ RUN apt-get update && apt-get install -y \
         texlive-latex-extra \
         texlive-latex-recommended \
         iproute2 \
-# For python-click build
-        python-sphinx \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -358,11 +348,6 @@ RUN pip3 uninstall -y enum34
 # For templating
 RUN pip install j2cli==0.3.10
 
-# Remove python-click 6.6
-RUN apt-get purge -y python-click
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 # For sonic snmpagent mock testing
 RUN pip3 install mockredispy==2.9.3
 

src/python-click/.gitignore

@@ -1,3 +0,0 @@
-*
-!.gitignore
-!Makefile

src/python-click/Makefile

@@ -1,26 +0,0 @@
-.ONESHELL:
-SHELL = /bin/bash
-.SHELLFLAGS += -e
-
-MAIN_TARGET = python-click_$(PYTHON_CLICK_VERSION)_all.deb
-
-$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
-	# Remove any stale files
-	rm -rf ./python-click
-
-	# Clone python-click Debian repo
-	git clone https://salsa.debian.org/debian/python-click
-
-	pushd ./python-click
-
-	# Reset HEAD to the commit of the proper tag
-	# NOTE: Using "git checkout <tag_name>" here detaches our HEAD,
-	# which stg doesn't like, so we use this method instead
-	git reset --hard debian/$(PYTHON_CLICK_VERSION)
-
-	# Build source and Debian packages
-	dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
-	popd
-
-	# Move the newly-built .deb package to the destination directory
-	mv $* $(DEST)/

src/sonic-py-common/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 
 dependencies = [
-    'natsort',
+    'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2
     'pyyaml',
     'swsssdk>=2.0.1',
 ]
@@ -28,7 +28,8 @@ setup(
         'sonic_py_common',
     ],
     setup_requires= [
-        'pytest-runner'
+        'pytest-runner',
+        'wheel'
     ],
     tests_require=[
         'pytest',