From d40c9a1e8ded033eb825c451bec6b4b822aed8db Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Fri, 25 Dec 2020 21:29:25 -0800 Subject: [PATCH] [docker-base-buster][docker-config-engine-buster] No longer install Python 2 (#6162) **- Why I did it** As part of migrating SONiC codebase from Python 2 to Python 3 **- How I did it** - No longer install Python 2 in docker-base-buster or docker-config-engine-buster. - Install Python 2 and pip2 in the following containers until we can completely eliminate it there: - docker-platform-monitor - docker-sonic-mgmt-framework - docker-sonic-vs - Pin pip2 version <21 where it is still temporarily needed, as pip version 21 will drop support for Python 2 - Also preform some other cleanup, ensuring that pip3, setuptools and wheel packages are installed in docker-base-buster, and then removing any attempts to re-install them in derived containers --- dockers/docker-base-buster/Dockerfile.j2 | 19 ++++++++-------- dockers/docker-base/Dockerfile.j2 | 3 +++ .../docker-config-engine-buster/Dockerfile.j2 | 22 ++++++------------- .../Dockerfile.j2 | 15 +++++-------- dockers/docker-config-engine/Dockerfile.j2 | 6 +---- .../docker-database/docker-database-init.sh | 2 +- dockers/docker-database/supervisord.conf.j2 | 2 +- dockers/docker-orchagent/Dockerfile.j2 | 10 --------- dockers/docker-platform-monitor/Dockerfile.j2 | 13 +++++++++-- dockers/docker-snmp/Dockerfile.j2 | 8 ++----- .../docker-sonic-mgmt-framework/Dockerfile.j2 | 13 +++++++++-- platform/vs/docker-sonic-vs/Dockerfile.j2 | 8 ++++--- rules/docker-config-engine-buster.mk | 3 --- rules/docker-platform-monitor.mk | 1 + sonic-slave-buster/Dockerfile.j2 | 2 +- sonic-slave-stretch/Dockerfile.j2 | 2 +- 16 files changed, 60 insertions(+), 69 deletions(-) diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index bf1d8197bf..1f80775f11 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -44,9 +44,8 @@ RUN apt-get update && \ less \ perl \ procps \ - python \ - python-pip \ python3 \ + python3-distutils \ python3-pip \ rsyslog \ vim-tiny \ @@ -57,7 +56,6 @@ RUN apt-get update && \ lua-bitop \ lua-cjson \ # common dependencies - libpython2.7 \ libdaemon0 \ libdbus-1-3 \ libjansson4 \ @@ -84,15 +82,16 @@ RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb || apt-get install -f -y RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb {% endif %} -# Some Python packages require setuptools (or pkg_resources, which is supplied by setuptools) -# and some require wheel -RUN pip install setuptools==40.8.0 -RUN pip3 install setuptools==49.6.00 -RUN pip install wheel -RUN pip3 install wheel +# Upgrade pip via PyPI and uninstall the Debian version +RUN pip3 install --upgrade pip +RUN apt-get purge -y python3-pip + +# setuptools and wheel are necessary for installing some Python wheel packages +RUN pip3 install --no-cache-dir setuptools==49.6.00 +RUN pip3 install --no-cache-dir wheel==0.35.1 # For templating -RUN pip2 install j2cli +RUN pip3 install j2cli # Install supervisor RUN pip3 install supervisor==4.2.1 diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2 index 591759bbca..fc293da9da 100644 --- a/dockers/docker-base/Dockerfile.j2 +++ b/dockers/docker-base/Dockerfile.j2 @@ -52,6 +52,9 @@ COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"] COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"] COPY ["root/.vimrc", "/root/.vimrc"] +RUN pip install --upgrade 'pip<21' +RUN apt-get purge -y python-pip + # Some Python packages require setuptools (or pkg_resources, which is supplied by setuptools) # and some require wheel RUN pip install setuptools==40.8.0 diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2 index df9c9f473c..3022546a06 100644 --- a/dockers/docker-config-engine-buster/Dockerfile.j2 +++ b/dockers/docker-config-engine-buster/Dockerfile.j2 @@ -6,22 +6,15 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y \ - build-essential \ - python-pip \ - python3-pip \ - python-dev \ - python3-dev \ apt-utils \ -{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %} - libxslt-dev \ - libz-dev \ -{%- endif %} - python-setuptools \ - python3-setuptools + build-essential \ + python3-dev -RUN pip3 install --upgrade pip -RUN pip2 install --upgrade pip -RUN apt-get purge -y python-pip python3-pip +{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %} +RUN apt-get install -y \ + libxslt-dev \ + libz-dev +{%- endif %} # For sonic-config-engine Python 3 package # Explicitly install pyangbind here, as pyangbind causes enum34 to be installed. @@ -51,7 +44,6 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] ## Clean up RUN apt-get purge -y \ - python-dev \ python3-dev \ build-essential && \ apt-get clean -y && \ diff --git a/dockers/docker-config-engine-stretch/Dockerfile.j2 b/dockers/docker-config-engine-stretch/Dockerfile.j2 index 66f47a02c1..813f41bc29 100644 --- a/dockers/docker-config-engine-stretch/Dockerfile.j2 +++ b/dockers/docker-config-engine-stretch/Dockerfile.j2 @@ -8,15 +8,13 @@ RUN apt-get update && \ apt-get install -y \ # Dependencies for sonic-cfggen build-essential \ - python-pip \ - python-dev \ -{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %} - libxslt-dev \ - libz-dev \ -{%- endif %} - python-setuptools + python-dev -RUN pip install --upgrade pip +{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %} +RUN apt-get install -y \ + libxslt-dev \ + libz-dev +{%- endif %} {% if docker_config_engine_stretch_debs.strip() %} # Copy locally-built Debian package dependencies @@ -40,7 +38,6 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] ## Clean up RUN apt-get purge -y \ build-essential \ - python-pip \ python-dev && \ apt-get clean -y && \ apt-get autoclean -y && \ diff --git a/dockers/docker-config-engine/Dockerfile.j2 b/dockers/docker-config-engine/Dockerfile.j2 index c5adf15cb7..8790ba67d7 100644 --- a/dockers/docker-config-engine/Dockerfile.j2 +++ b/dockers/docker-config-engine/Dockerfile.j2 @@ -6,11 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update # Dependencies for sonic-cfggen -RUN apt-get install -y build-essential python-pip python-dev - -RUN pip install --upgrade pip - -RUN apt-get purge -y python-pip +RUN apt-get install -y build-essential python-dev {% if docker_config_engine_debs.strip() %} COPY \ diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index 0fd86ee3df..413bbc0b20 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -79,7 +79,7 @@ rm $db_cfg_file_tmp # copy dump.rdb file to each instance for restoration DUMPFILE=/var/lib/redis/dump.rdb -redis_inst_list=`/usr/bin/python -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"` +redis_inst_list=`/usr/bin/python3 -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"` for inst in $redis_inst_list do mkdir -p /var/lib/$inst diff --git a/dockers/docker-database/supervisord.conf.j2 b/dockers/docker-database/supervisord.conf.j2 index a1d10f78bd..65a172b374 100644 --- a/dockers/docker-database/supervisord.conf.j2 +++ b/dockers/docker-database/supervisord.conf.j2 @@ -18,7 +18,7 @@ stdout_logfile=syslog stderr_logfile=syslog {% if INSTANCES %} -{% for redis_inst, redis_items in INSTANCES.iteritems() %} +{% for redis_inst, redis_items in INSTANCES.items() %} [program: {{ redis_inst }}] {% if redis_items['hostname'] != '127.0.0.1' and redis_inst != 'redis_chassis' %} {%- set LOOPBACK_IP = '127.0.0.1' -%} diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2 index 4d5c0f0fd5..17c0983c58 100755 --- a/dockers/docker-orchagent/Dockerfile.j2 +++ b/dockers/docker-orchagent/Dockerfile.j2 @@ -21,7 +21,6 @@ RUN apt-get update && \ ndppd \ # Needed for installing netifaces Python package build-essential \ - python-dev \ python3-dev {% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %} @@ -36,14 +35,6 @@ RUN ln -s -f /usr/bin/gcc-8 /usr/bin/arm-linux-gnueabihf-gcc RUN ln -s -f /usr/bin/gcc-8 /usr/bin/aarch64-linux-gnu-gcc {% endif %} -RUN pip2 install \ - scapy==2.4.2 \ - setuptools \ - pyroute2==0.5.3 -RUN pip2 install \ - netifaces==0.10.7 \ - monotonic==1.5 - # Dependencies of restore_neighbors.py RUN pip3 install \ scapy==2.4.4 \ @@ -66,7 +57,6 @@ RUN apt-get remove -y gcc-8 # Clean up RUN apt-get purge -y \ build-essential \ - python-dev \ python3-dev && \ apt-get clean -y && \ apt-get autoclean -y && \ diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2 index 1aed981db1..fd65763bf1 100755 --- a/dockers/docker-platform-monitor/Dockerfile.j2 +++ b/dockers/docker-platform-monitor/Dockerfile.j2 @@ -14,7 +14,7 @@ ENV IMAGE_VERSION=$image_version # Install required packages RUN apt-get update && \ apt-get install -y \ - python-dev \ + build-essential \ python3-dev \ ipmitool \ librrd8 \ @@ -26,6 +26,11 @@ RUN apt-get update && \ i2c-tools \ python-jsonschema +# TODO: Remove these lines once we no longer need Python 2 +RUN apt-get install -f -y python-dev python-pip +RUN pip2 install --upgrade 'pip<21' +RUN apt-get purge -y python-pip + # On Arista devices, the sonic_platform wheel is not installed in the container. # Instead, the installation directory is mounted from the host OS. However, this method # doesn't ensure all dependencies are installed in the container. So here we @@ -64,9 +69,13 @@ RUN pip3 install thrift==0.13.0 {{ install_python_wheels(docker_platform_monitor_whls.split(' ')) }} {% endif %} + +# TODO: Remove this line once we no longer need Python 2 +RUN apt-get purge -y python-dev + # Clean up RUN apt-get purge -y \ - python-dev \ + build-essential \ python3-dev && \ apt-get clean -y && \ apt-get autoclean -y && \ diff --git a/dockers/docker-snmp/Dockerfile.j2 b/dockers/docker-snmp/Dockerfile.j2 index 2fdde61252..e27987dc0d 100644 --- a/dockers/docker-snmp/Dockerfile.j2 +++ b/dockers/docker-snmp/Dockerfile.j2 @@ -40,12 +40,8 @@ ENV LC_CTYPE=en_US.UTF-8 RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen RUN locale-gen -# Install up-to-date version of pip -RUN pip3 install --no-cache-dir setuptools wheel - -# Install pyyaml dependency for use by some plugins -# Install smbus dependency for use by some plugins -RUN python3 -m pip install --no-cache-dir \ +# Install dependencies used by some plugins +RUN pip3 install --no-cache-dir \ hiredis \ pyyaml \ smbus diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index b258a40ee3..5ee8cae844 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -7,8 +7,14 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y g++ python-dev python3-dev libxml2 + apt-get install -y g++ python3-dev libxml2 +# TODO: Remove these lines once we no longer need Python 2 +RUN apt-get install -f -y python-dev python-pip +RUN pip2 install --upgrade 'pip<21' +RUN apt-get purge -y python-pip +RUN pip2 install setuptools==40.8.0 +RUN pip2 install wheel==0.35.1 RUN pip2 install connexion==1.1.15 \ setuptools==21.0.0 \ grpcio-tools==1.20.0 \ @@ -40,7 +46,10 @@ COPY ["start.sh", "rest-server.sh", "/usr/bin/"] COPY ["mgmt_vars.j2", "/usr/share/sonic/templates/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] -RUN apt-get remove -y g++ python-dev python3-dev +# TODO: Remove this line once we no longer need Python 2 +RUN apt-get purge -y python-dev + +RUN apt-get remove -y g++ python3-dev RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y RUN rm -rf /debs diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2 index 536eec065c..a9f05f9b74 100644 --- a/platform/vs/docker-sonic-vs/Dockerfile.j2 +++ b/platform/vs/docker-sonic-vs/Dockerfile.j2 @@ -42,13 +42,13 @@ RUN apt-get install -y net-tools \ python-scapy \ conntrack \ iptables \ - python3-pip \ jq \ libzmq5 \ # For installing Python m2crypto package # (these can be uninstalled after installation) build-essential \ python-dev \ + python-pip \ python3-dev \ libssl-dev \ swig \ @@ -73,8 +73,10 @@ RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_a RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb {% endif %} -RUN pip2 install setuptools -RUN pip3 install setuptools +RUN pip2 install --upgrade 'pip<21' +RUN apt-get purge -y python-pip +RUN pip2 install setuptools==40.8.0 +RUN pip2 install wheel==0.33.6 RUN pip2 install py2_ipaddress RUN pip2 install six RUN pip2 install pyroute2==0.5.3 netifaces==0.10.7 diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index a3d037b9ca..b386c882f4 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -4,11 +4,8 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz $(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster $(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) -$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3) -$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) -$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk index 1bd6a68660..7ab0f2812c 100644 --- a/rules/docker-platform-monitor.mk +++ b/rules/docker-platform-monitor.mk @@ -14,6 +14,7 @@ $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY3) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) +$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY3) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_LEDD) diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 81e56ccac7..9069805b7b 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -370,7 +370,7 @@ RUN export VERSION=1.14.2 \ && rm go$VERSION.linux-*.tar.gz RUN pip3 install --upgrade pip -RUN pip2 install --upgrade pip +RUN pip2 install --upgrade 'pip<21' RUN apt-get purge -y python-pip python3-pip # For building Python packages diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index c1e9d10951..0c7f6688fe 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -339,7 +339,7 @@ RUN export VERSION=1.14.2 \ && rm go$VERSION.linux-*.tar.gz RUN pip3 install --upgrade pip -RUN pip2 install --upgrade pip +RUN pip2 install --upgrade 'pip<21' RUN apt-get purge -y python-pip python3-pip # For p4 build