From a07aaca831c588067d87f9f9823acd1b9a7daa91 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 18 Oct 2022 10:02:30 +0800 Subject: [PATCH] [docker-sonic-mgmt] Cleanup and upgrade some packages (#12218) Why I did it The Dockerfile of docker-sonic-mgmt became a little bit messy over time. Some packages are also a little bit too old. It would be better to do some cleanup and upgrade some important packages. How I did it Updated the dockerfile template for building docker-sonic-mgmt. How to verify it Locally built the docker-sonic-mgmt image and used it to run some test scripts. Description for the changelog: The build-essential package contains gcc and make. It's unnecessary to install them again. The python-is-python2 package is included in the python package for Ubuntu 20.04. It's unnecessary to install it again. Sort the apt and pip packages by alphabetic order. Cleanup get-pip.py after installation. Cleanup the python-scapy deb package after installation. Ensure that the python pip, setuptools and wheel packages are up to date. Install pytest-ansible from pip instead of from source code. While installing docker-ce-cli, it's unnecessary to install curl and software-properties-common again. Merged some pip install steps into one step. Upgrade ansible from 2.8.12 to 2.9.27 for env-python3. Upgrade pytest to 7.1.3 for env-python3. Add ncclient package to evn-python3. --- dockers/docker-sonic-mgmt/Dockerfile.j2 | 196 +++++++++++------------- 1 file changed, 90 insertions(+), 106 deletions(-) diff --git a/dockers/docker-sonic-mgmt/Dockerfile.j2 b/dockers/docker-sonic-mgmt/Dockerfile.j2 index 627fbaa65c..77b30ef784 100755 --- a/dockers/docker-sonic-mgmt/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt/Dockerfile.j2 @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y build-essential \ cmake \ curl \ default-jre \ - gcc \ git \ inetutils-ping \ iproute2 \ @@ -16,7 +15,6 @@ RUN apt-get update && apt-get install -y build-essential \ libssl-dev \ libxml2 \ libxslt1-dev \ - make \ openssh-server \ psmisc \ python \ @@ -25,36 +23,43 @@ RUN apt-get update && apt-get install -y build-essential \ python3-venv \ rsyslog \ snmp \ + software-properties-common \ sshpass \ sudo \ tcpdump \ telnet \ - vim \ - python-is-python2 \ - software-properties-common + vim -RUN add-apt-repository -y universe -RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ - && python2 get-pip.py - -RUN curl -L http://archive.ubuntu.com/ubuntu/pool/universe/s/scapy/python-scapy_2.3.3-3_all.deb \ +RUN curl -fsSL http://archive.ubuntu.com/ubuntu/pool/universe/s/scapy/python-scapy_2.3.3-3_all.deb \ --output python-scapy_2.3.3-3_all.deb \ - && dpkg -i python-scapy_2.3.3-3_all.deb + && dpkg -i python-scapy_2.3.3-3_all.deb \ + && rm -f python-scapy_2.3.3-3_all.deb -RUN pip install setuptools==44.1.1 -RUN pip install cffi==1.12.0 \ +RUN curl -fsSL https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ + && python2 get-pip.py \ + && rm -f get-pip.py \ + && ln -sf `which pip2` /usr/bin/pip + +RUN pip install --upgrade pip setuptools wheel +RUN pip install allure-pytest==2.8.22 \ + ansible==2.8.12 \ + azure-storage-blob==12.9.0 \ + celery[redis]==4.4.7 \ + cffi==1.12.0 \ contextlib2==0.6.0.post1 \ cryptography==3.3.2 \ + dpkt \ "future>=0.16.0" \ gitpython \ ipaddr \ ipython==5.4.1 \ ixnetwork-restpy==1.0.64 \ ixnetwork-open-traffic-generator==0.0.79 \ - snappi[ixnetwork,convergence]==0.7.44 \ jinja2==2.7.2 \ jsonpatch \ + lazy-object-proxy==1.6.0 \ lxml \ + msrest==0.6.21 \ natsort \ netaddr \ netmiko==2.4.2 \ @@ -64,30 +69,29 @@ RUN pip install cffi==1.12.0 \ prettytable \ psutil \ pyasn1==0.1.9 \ + pycryptodome==3.9.8 \ pyfiglet \ - lazy-object-proxy==1.6.0 \ pylint==1.8.1 \ pyro4 \ pysnmp==4.2.5 \ + pysubnettree \ + pytest==4.6.11 \ + pytest-ansible \ pytest-repeat \ pytest-html \ pytest-xdist==1.28.0 \ - pytest==4.6.5 \ + python-dateutil \ redis \ requests \ + retry \ rpyc \ six \ - tabulate \ + snappi[ixnetwork,convergence]==0.7.44 \ statistics \ - textfsm==1.1.2 \ - virtualenv \ - retry \ + tabulate \ + textfsm==1.1.3 \ thrift==0.11.0 \ - allure-pytest==2.8.22 \ - celery[redis]==4.4.7 \ - msrest==0.6.21 \ - python-dateutil \ - azure-storage-blob==12.9.0 \ + virtualenv \ && git clone https://github.com/p4lang/scapy-vxlan.git \ && cd scapy-vxlan \ && python setup.py install \ @@ -104,8 +108,7 @@ RUN pip install cffi==1.12.0 \ && cd ../.. \ && rm -fr nanomsg-1.0.0 \ && rm -f 1.0.0.tar.gz \ - && pip install nnpy \ - && pip install dpkt \ + && pip install nnpy \ && pip install scapy==2.4.5 --upgrade --ignore-installed # Install docker-ce-cli @@ -113,9 +116,7 @@ RUN apt-get update \ && apt-get install -y \ apt-transport-https \ ca-certificates \ - curl \ gnupg-agent \ - software-properties-common \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ && apt-get update \ @@ -124,8 +125,6 @@ RUN apt-get update \ # Install Azure CLI RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash -RUN pip install wheel==0.33.6 - ## Copy and install sonic-mgmt docker dependencies COPY \ {% for deb in docker_sonic_mgmt_debs.split(' ') -%} @@ -138,18 +137,6 @@ RUN dpkg -i \ debs/{{ deb }}{{' '}} {%- endfor %} -RUN pip install ansible==2.8.12 - -RUN pip install pysubnettree - -# Install pytest-ansible module with 'become', 'become_user' parameters support -RUN git clone https://github.com/ansible/pytest-ansible.git \ - && cd pytest-ansible \ - && git checkout d33c025f070a9c870220a157cc5a999fda68de44 \ - && python setup.py install \ - && cd .. \ - && rm -fr pytest-ansible - RUN mkdir /var/run/sshd EXPOSE 22 @@ -188,10 +175,6 @@ RUN env-201811/bin/pip install cryptography==3.3.2 ansible==2.0.0.2 RUN python3 -m venv env-python3 -# NOTE: There is an ordering dependency for pycryptodome. Leaving this at -# the end until we figure that out. -RUN pip install pycryptodome==3.9.8 - # Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD. ENV VIRTUAL_ENV=env-python3 ARG BACKUP_OF_PATH="$PATH" @@ -199,65 +182,66 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 -RUN python3 -m pip install --upgrade --ignore-installed pip setuptools==58.4.0 wheel==0.33.6 -RUN python3 -m pip install setuptools-rust \ - aiohttp \ - defusedxml \ - azure-kusto-ingest \ - azure-kusto-data \ - cffi \ - contextlib2==0.6.0.post1 \ - cryptography==3.3.2 \ - "future>=0.16.0" \ - gitpython \ - ipaddr \ - ipython==5.4.1 \ - ixnetwork-restpy==1.0.64 \ - ixnetwork-open-traffic-generator==0.0.79 \ - snappi[ixnetwork,convergence]==0.7.44 \ - markupsafe==2.0.1 \ - jinja2==2.7.2 \ - jsonpatch \ - lxml \ - natsort \ - netaddr \ - netmiko==2.4.2 \ - paramiko==2.7.1 \ - passlib \ - pexpect \ - prettytable \ - psutil \ - pyasn1==0.4.8 \ - pyfiglet \ - pylint==1.8.1 \ - pyro4 \ - pysnmp==4.4.12 \ - pytest-repeat \ - pytest-html \ - pytest-xdist==1.28.0 \ - pytest \ - redis \ - requests \ - rpyc \ - six \ - tabulate \ - textfsm==1.1.2 \ - virtualenv \ - pysubnettree \ - nnpy \ - dpkt \ - pycryptodome==3.9.8 \ - ansible==2.8.12 \ - pytest-ansible \ - allure-pytest==2.8.22 \ - retry \ - thrift==0.11.0 \ - ptf \ - scapy==2.4.5 \ - celery[redis]==4.4.7 \ - msrest==0.6.21 \ - python-dateutil \ - azure-storage-blob==12.9.0 +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install aiohttp \ + allure-pytest==2.8.22 \ + ansible==2.9.27 \ + azure-storage-blob==12.9.0 \ + azure-kusto-data \ + azure-kusto-ingest \ + defusedxml \ + celery[redis]==4.4.7 \ + cffi \ + contextlib2==0.6.0.post1 \ + cryptography==3.3.2 \ + dpkt \ + "future>=0.16.0" \ + gitpython \ + ipaddr \ + ipython==5.4.1 \ + ixnetwork-restpy==1.0.64 \ + ixnetwork-open-traffic-generator==0.0.79 \ + jinja2==2.7.2 \ + jsonpatch \ + lxml \ + markupsafe==2.0.1 \ + msrest==0.6.21 \ + natsort \ + ncclient \ + netaddr \ + netmiko==2.4.2 \ + nnpy \ + paramiko==2.7.1 \ + passlib \ + pexpect \ + prettytable \ + psutil \ + ptf \ + pyasn1==0.4.8 \ + pycryptodome==3.9.8 \ + pyfiglet \ + pylint==1.8.1 \ + pyro4 \ + pysnmp==4.4.12 \ + pysubnettree \ + pytest-ansible \ + pytest-html \ + pytest-repeat \ + pytest-xdist==1.28.0 \ + python-dateutil \ + pytest==7.1.3 \ + redis \ + requests \ + retry \ + rpyc \ + scapy==2.4.5 \ + setuptools-rust \ + six \ + snappi[ixnetwork,convergence]==0.7.44 \ + tabulate \ + textfsm==1.1.2 \ + thrift==0.11.0 \ + virtualenv # Deactivating a virtualenv ENV PATH="$BACKUP_OF_PATH"