From 4426f7715fd4f5442e63e5efe759ad86006e9196 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:23:35 +0300 Subject: [PATCH] [scapy] update scapy to 2.4.5 and patch it (#10457) Why I did it Running warm-reboot in a loop for 500 times leads to this error on 318-th iteration: Apr 2 15:56:27.346747 sonic INFO swss#/supervisord: restore_neighbors Traceback (most recent call last): Apr 2 15:56:27.346747 sonic INFO swss#/supervisord: restore_neighbors File "/usr/bin/restore_neighbors.py", line 24, in Apr 2 15:56:27.346747 sonic INFO swss#/supervisord: restore_neighbors from scapy.all import conf, in6_getnsma, inet_pton, inet_ntop, in6_getnsmac, get_if_hwaddr, Ether, ARP, IPv6, ICMPv6ND_NS, ICMPv6NDOptSrcLLAddr Apr 2 15:56:27.346795 sonic INFO swss#/supervisord: restore_neighbors File "/usr/local/lib/python3.7/dist-packages/scapy/all.py", line 25, in Apr 2 15:56:27.346956 sonic INFO swss#/supervisord: restore_neighbors from scapy.route import * Apr 2 15:56:27.346995 sonic INFO swss#/supervisord: restore_neighbors File "/usr/local/lib/python3.7/dist-packages/scapy/route.py", line 205, in Apr 2 15:56:27.347089 sonic INFO swss#/supervisord: restore_neighbors conf.iface = get_working_if() Apr 2 15:56:27.347129 sonic INFO swss#/supervisord: restore_neighbors File "/usr/local/lib/python3.7/dist-packages/scapy/arch/linux.py", line 128, in get_working_if Apr 2 15:56:27.347213 sonic INFO swss#/supervisord: restore_neighbors ifflags = struct.unpack("16xH14x", get_if(i, SIOCGIFFLAGS))[0] Apr 2 15:56:27.347250 sonic INFO swss#/supervisord: restore_neighbors File "/usr/local/lib/python3.7/dist-packages/scapy/arch/common.py", line 31, in get_if Apr 2 15:56:27.347345 sonic INFO swss#/supervisord: restore_neighbors return ioctl(sck, cmd, struct.pack("16s16x", iff.encode("utf8"))) Apr 2 15:56:27.347365 sonic INFO swss#/supervisord: restore_neighbors OSError: [Errno 19] No such device The issue was reported to scapy devs secdev/scapy#3369, the fix is secdev/scapy#3371, however there is no released scapy version with this fix right now, thus decided to build scapy v2.4.5 from sources and apply the fix in a form of a patch. Signed-off-by: Stepan Blyschak --- .gitmodules | 3 ++ dockers/docker-orchagent/Dockerfile.j2 | 9 +++- rules/docker-orchagent.mk | 1 + rules/scapy.dep | 10 +++++ rules/scapy.mk | 7 ++++ src/scapy | 1 + ...ng-generation-when-scapy-is-a-submod.patch | 27 ++++++++++++ ...f-the-network-interface-still-exists.patch | 42 +++++++++++++++++++ src/scapy.patch/series | 2 + 9 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 rules/scapy.dep create mode 100644 rules/scapy.mk create mode 160000 src/scapy create mode 100644 src/scapy.patch/0001-Fix-version-string-generation-when-scapy-is-a-submod.patch create mode 100644 src/scapy.patch/0002-Check-if-the-network-interface-still-exists.patch create mode 100644 src/scapy.patch/series diff --git a/.gitmodules b/.gitmodules index f79031a7d2..b2d76bd66b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -62,6 +62,9 @@ [submodule "src/redis-dump-load"] path = src/redis-dump-load url = https://github.com/p/redis-dump-load.git +[submodule "src/scapy"] + path = src/scapy + url = https://github.com/secdev/scapy.git [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"] path = platform/mellanox/mlnx-sai/SAI-Implementation url = https://github.com/Mellanox/SAI-Implementation diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2 index f71f31cfc0..ea4e0abbc2 100755 --- a/dockers/docker-orchagent/Dockerfile.j2 +++ b/dockers/docker-orchagent/Dockerfile.j2 @@ -40,7 +40,6 @@ RUN ln -s -f /usr/bin/gcc-8 /usr/bin/aarch64-linux-gnu-gcc # Dependencies of restore_neighbors.py RUN pip3 install \ - scapy==2.4.4 \ pyroute2==0.5.14 \ netifaces==0.10.9 @@ -57,6 +56,14 @@ RUN apt-get remove -y gcc-8 {{ install_debian_packages(docker_orchagent_debs.split(' ')) }} {%- endif %} +{% if docker_orchagent_whls.strip() -%} +# Copy locally-built Python wheel dependencies +{{ copy_files("python-wheels/", docker_orchagent_whls.split(' '), "/python-wheels/") }} + +# Install locally-built Python wheel dependencies +{{ install_python_wheels(docker_orchagent_whls.split(' ')) }} +{% endif %} + # Clean up RUN apt-get purge -y \ build-essential \ diff --git a/rules/docker-orchagent.mk b/rules/docker-orchagent.mk index d9453465c7..ef0c203877 100644 --- a/rules/docker-orchagent.mk +++ b/rules/docker-orchagent.mk @@ -10,6 +10,7 @@ $(DOCKER_ORCHAGENT)_DBG_DEPENDS = $($(DOCKER_SWSS_LAYER_BUSTER)_DBG_DEPENDS) $(DOCKER_ORCHAGENT)_DBG_DEPENDS += $(SWSS_DBG) \ $(LIBSWSSCOMMON_DBG) \ $(LIBSAIREDIS_DBG) +$(DOCKER_ORCHAGENT)_PYTHON_WHEELS += $(SCAPY) $(DOCKER_ORCHAGENT)_DBG_IMAGE_PACKAGES = $($(DOCKER_SWSS_LAYER_BUSTER)_DBG_IMAGE_PACKAGES) diff --git a/rules/scapy.dep b/rules/scapy.dep new file mode 100644 index 0000000000..ef09fb1222 --- /dev/null +++ b/rules/scapy.dep @@ -0,0 +1,10 @@ +SPATH := $($(SCAPY)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/scapy.mk rules/scapy.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files -- ':!:doc/*')) + +$(SCAPY)_CACHE_MODE := GIT_CONTENT_SHA +$(SCAPY)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SCAPY)_DEP_FILES := $(DEP_FILES) +$(SCAPY)_SMDEP_FILES := $(SMDEP_FILES) +$(SCAPY)_SMDEP_PATHS := $(SPATH) diff --git a/rules/scapy.mk b/rules/scapy.mk new file mode 100644 index 0000000000..6953c1ef10 --- /dev/null +++ b/rules/scapy.mk @@ -0,0 +1,7 @@ +# scapy python3 wheel + +SCAPY = scapy-2.4.5-py2.py3-none-any.whl +$(SCAPY)_SRC_PATH = $(SRC_PATH)/scapy +$(SCAPY)_PYTHON_VERSION = 3 +$(SCAPY)_TEST = n +SONIC_PYTHON_WHEELS += $(SCAPY) diff --git a/src/scapy b/src/scapy new file mode 160000 index 0000000000..8b63d73a17 --- /dev/null +++ b/src/scapy @@ -0,0 +1 @@ +Subproject commit 8b63d73a17266bae2a61513ea97ded5283a7ccd3 diff --git a/src/scapy.patch/0001-Fix-version-string-generation-when-scapy-is-a-submod.patch b/src/scapy.patch/0001-Fix-version-string-generation-when-scapy-is-a-submod.patch new file mode 100644 index 0000000000..37f5253db0 --- /dev/null +++ b/src/scapy.patch/0001-Fix-version-string-generation-when-scapy-is-a-submod.patch @@ -0,0 +1,27 @@ +From 988c808af6065b740006aef0e94496821d41fb98 Mon Sep 17 00:00:00 2001 +From: Stepan Blyschak +Date: Mon, 4 Apr 2022 09:57:39 +0000 +Subject: [PATCH] Fix version string generation when scapy is a submodule + +Signed-off-by: Stepan Blyschak +--- + scapy/__init__.py | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/scapy/__init__.py b/scapy/__init__.py +index f920151e..2adbea19 100644 +--- a/scapy/__init__.py ++++ b/scapy/__init__.py +@@ -42,9 +42,6 @@ def _version_from_git_describe(): + :raises CalledProcessError: if git is unavailable + :return: Scapy's latest tag + """ +- if not os.path.isdir(os.path.join(os.path.dirname(_SCAPY_PKG_DIR), '.git')): # noqa: E501 +- raise ValueError('not in scapy git repo') +- + def _git(cmd): + # type: (str) -> str + process = subprocess.Popen( +-- +2.14.1 + diff --git a/src/scapy.patch/0002-Check-if-the-network-interface-still-exists.patch b/src/scapy.patch/0002-Check-if-the-network-interface-still-exists.patch new file mode 100644 index 0000000000..c42c12e1a8 --- /dev/null +++ b/src/scapy.patch/0002-Check-if-the-network-interface-still-exists.patch @@ -0,0 +1,42 @@ +From 7ffd8101c1e535f9c3225db2c319958a64412686 Mon Sep 17 00:00:00 2001 +From: Guillaume Valadon +Date: Tue, 14 Sep 2021 19:34:43 +0200 +Subject: [PATCH] Check if the network interface still exists + +Signed-off-by: Stepan Blyschak +--- + scapy/arch/linux.py | 18 +++++++++++------- + 1 file changed, 11 insertions(+), 7 deletions(-) + +diff --git a/scapy/arch/linux.py b/scapy/arch/linux.py +index 94fac8f0..b86e98ab 100644 +--- a/scapy/arch/linux.py ++++ b/scapy/arch/linux.py +@@ -414,13 +414,17 @@ class LinuxInterfaceProvider(InterfaceProvider): + data = {} + ips = in6_getifaddr() + for i in _get_if_list(): +- ifflags = struct.unpack("16xH14x", get_if(i, SIOCGIFFLAGS))[0] +- index = get_if_index(i) +- mac = scapy.utils.str2mac( +- get_if_raw_hwaddr(i, siocgifhwaddr=SIOCGIFHWADDR)[1] +- ) +- ip = None # type: Optional[str] +- ip = inet_ntop(socket.AF_INET, get_if_raw_addr(i)) ++ try: ++ ifflags = struct.unpack("16xH14x", get_if(i, SIOCGIFFLAGS))[0] ++ index = get_if_index(i) ++ mac = scapy.utils.str2mac( ++ get_if_raw_hwaddr(i, siocgifhwaddr=SIOCGIFHWADDR)[1] ++ ) ++ ip = None # type: Optional[str] ++ ip = inet_ntop(socket.AF_INET, get_if_raw_addr(i)) ++ except IOError: ++ warning("Interface %s does not exist!", i) ++ continue + if ip == "0.0.0.0": + ip = None + ifflags = FlagValue(ifflags, _iff_flags) +-- +2.14.1 + diff --git a/src/scapy.patch/series b/src/scapy.patch/series new file mode 100644 index 0000000000..d64f20f136 --- /dev/null +++ b/src/scapy.patch/series @@ -0,0 +1,2 @@ +0001-Fix-version-string-generation-when-scapy-is-a-submod.patch +0002-Check-if-the-network-interface-still-exists.patch