From 65e7a84509d39ee14fd7e4d20ddfff56921359d3 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 30 May 2020 05:52:27 -0700 Subject: [PATCH] [baseimage]: Build and install redis-dump-load Python 3 package in host image (#4661) Fix #4656 --- .../build_templates/sonic_debian_extension.j2 | 20 +++++++++++++------ rules/redis-dump-load-py3.mk | 9 +++++++++ rules/swss-common.mk | 2 +- rules/swsssdk-py2.mk | 1 + rules/swsssdk-py3.mk | 2 +- slave.mk | 1 + .../0001-Use-pipelines-when-dumping-52.patch | 9 +++++---- 7 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 rules/redis-dump-load-py3.mk diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 398fa173e3..e88c54692f 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -101,6 +101,13 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME # Install Python client for Redis sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6" +# Install redis-dump-load Python 3 package +# Note: the scripts will be overwritten by corresponding Python 2 package +REDIS_DUMP_LOAD_PY3_WHEEL_NAME=$(basename {{redis_dump_load_py3_wheel_path}}) +sudo cp {{redis_dump_load_py3_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $REDIS_DUMP_LOAD_PY3_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME + # Install redis-dump-load Python 2 package REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}}) sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME @@ -110,13 +117,8 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME # Install Python module for ipaddress sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress -# Install SwSS SDK Python 2 package -SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}}) -sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME -sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME - # Install SwSS SDK Python 3 package +# Note: the scripts will be overwritten by corresponding Python 2 package if [ -e {{swsssdk_py3_wheel_path}} ]; then SWSSSDK_PY3_WHEEL_NAME=$(basename {{swsssdk_py3_wheel_path}}) sudo cp {{swsssdk_py3_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME @@ -124,6 +126,12 @@ if [ -e {{swsssdk_py3_wheel_path}} ]; then sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME fi +# Install SwSS SDK Python 2 package +SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}}) +sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME + # Install sonic-yang-models py3 package, install dependencies sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb diff --git a/rules/redis-dump-load-py3.mk b/rules/redis-dump-load-py3.mk new file mode 100644 index 0000000000..875b5e1c81 --- /dev/null +++ b/rules/redis-dump-load-py3.mk @@ -0,0 +1,9 @@ +# redis_dump_load python3 wheel + +REDIS_DUMP_LOAD_PY3 = redis_dump_load-1.1-py3-none-any.whl +$(REDIS_DUMP_LOAD_PY3)_SRC_PATH = $(SRC_PATH)/redis-dump-load +$(REDIS_DUMP_LOAD_PY3)_PYTHON_VERSION = 3 +# Synthetic dependency just to avoid race condition +$(REDIS_DUMP_LOAD_PY3)_DEPENDS += $(REDIS_DUMP_LOAD_PY2) +$(REDIS_DUMP_LOAD_PY3)_TEST = n +SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY3) diff --git a/rules/swss-common.mk b/rules/swss-common.mk index 2d49867f93..12d91e73fb 100644 --- a/rules/swss-common.mk +++ b/rules/swss-common.mk @@ -4,7 +4,7 @@ LIBSWSSCOMMON = libswsscommon_1.0.0_$(CONFIGURED_ARCH).deb $(LIBSWSSCOMMON)_SRC_PATH = $(SRC_PATH)/sonic-swss-common $(LIBSWSSCOMMON)_DEPENDS += $(LIBHIREDIS_DEV) $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \ $(LIBNL_ROUTE3_DEV) $(LIBNL_NF3_DEV) \ - $(LIBNL_CLI_DEV) $(SWIG) + $(LIBNL_CLI_DEV) $(SWIG) $(LIBSWSSCOMMON)_RDEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \ $(LIBNL_ROUTE3) $(LIBNL_NF3) $(LIBNL_CLI) SONIC_DPKG_DEBS += $(LIBSWSSCOMMON) diff --git a/rules/swsssdk-py2.mk b/rules/swsssdk-py2.mk index a45677ae58..8a98d1feda 100644 --- a/rules/swsssdk-py2.mk +++ b/rules/swsssdk-py2.mk @@ -3,4 +3,5 @@ SWSSSDK_PY2 = swsssdk-2.0.1-py2-none-any.whl $(SWSSSDK_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk $(SWSSSDK_PY2)_PYTHON_VERSION = 2 +$(SWSSSDK_PY2)_DEPENDS += $(REDIS_DUMP_LOAD_PY2) SONIC_PYTHON_WHEELS += $(SWSSSDK_PY2) diff --git a/rules/swsssdk-py3.mk b/rules/swsssdk-py3.mk index e2f3519e00..681ba72f85 100644 --- a/rules/swsssdk-py3.mk +++ b/rules/swsssdk-py3.mk @@ -4,5 +4,5 @@ SWSSSDK_PY3 = swsssdk-2.0.1-py3-none-any.whl $(SWSSSDK_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk $(SWSSSDK_PY3)_PYTHON_VERSION = 3 # Synthetic dependency just to avoid race condition -$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) +$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) $(REDIS_DUMP_LOAD_PY3) SONIC_PYTHON_WHEELS += $(SWSSSDK_PY3) diff --git a/slave.mk b/slave.mk index 34395848a3..06736eafe4 100644 --- a/slave.mk +++ b/slave.mk @@ -816,6 +816,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))" export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))" export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))" + export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))" export install_debug_image="$(INSTALL_DEBUG_TOOLS)" export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))" export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))" diff --git a/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch b/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch index 5eea3faba2..c37cbd9f4b 100644 --- a/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch +++ b/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch @@ -33,7 +33,7 @@ index df2870b..1b6063b 100755 if pretty: # hack to avoid implementing pretty printing fp.write(dumps(host=host, port=port, password=password, db=db, -@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding): +@@ -276,28 +276,77 @@ def _read_key(key, r, pretty, encoding): return (type, ttl, value) def _reader(r, pretty, encoding, keys='*'): @@ -68,7 +68,7 @@ index df2870b..1b6063b 100755 + i += 10000 + +def _read_keys(r, encoded_keys, pretty, encoding): -+ decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys] ++ decoded_keys = [encoded_key.decode(encoding) for encoded_key in encoded_keys] + do_keys = decoded_keys + retries = 5 + type_results = None @@ -84,12 +84,13 @@ index df2870b..1b6063b 100755 + p = r.pipeline() + for key in do_keys: + p.type(key) -+ type_results = p.execute() ++ encoded_type_results = p.execute() ++ type_results = [encoded_type_result.decode('ascii') for encoded_type_result in encoded_type_results] + + p = r.pipeline() + for i in range(len(do_keys)): + key = decoded_keys[i] -+ type = type_results[i].decode('ascii') ++ type = type_results[i] + if type == 'none': + # key was deleted by a concurrent operation on the data store. + # issue noops so that the number of results does not change