matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[baseimage]: Build and install redis-dump-load Python 3 package in host image (#4661)

Browse Source 
Fix #4656
This commit is contained in:
Qi Luo 2020-05-30 05:52:27 -07:00 committed by GitHub
parent d86af98930
commit 65e7a84509
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 32 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
files/build_templates/sonic_debian_extension.j2
View File

@ -101,6 +101,13 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
# Install Python client for Redis
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"
# Install redis-dump-load Python 3 package
# Note: the scripts will be overwritten by corresponding Python 2 package
REDIS_DUMP_LOAD_PY3_WHEEL_NAME=$(basename {{redis_dump_load_py3_wheel_path}})
sudo cp {{redis_dump_load_py3_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $REDIS_DUMP_LOAD_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
# Install redis-dump-load Python 2 package
REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
@ -110,13 +117,8 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
# Install Python module for ipaddress
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress
# Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
# Install SwSS SDK Python 3 package
# Note: the scripts will be overwritten by corresponding Python 2 package
if [ -e {{swsssdk_py3_wheel_path}} ]; then
SWSSSDK_PY3_WHEEL_NAME=$(basename {{swsssdk_py3_wheel_path}})
sudo cp {{swsssdk_py3_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
@ -124,6 +126,12 @@ if [ -e {{swsssdk_py3_wheel_path}} ]; then
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
fi
# Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
# Install sonic-yang-models py3 package, install dependencies
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb

9
rules/redis-dump-load-py3.mk Normal file
View File

@ -0,0 +1,9 @@
# redis_dump_load python3 wheel
REDIS_DUMP_LOAD_PY3 = redis_dump_load-1.1-py3-none-any.whl
$(REDIS_DUMP_LOAD_PY3)_SRC_PATH = $(SRC_PATH)/redis-dump-load
$(REDIS_DUMP_LOAD_PY3)_PYTHON_VERSION = 3
# Synthetic dependency just to avoid race condition
$(REDIS_DUMP_LOAD_PY3)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
$(REDIS_DUMP_LOAD_PY3)_TEST = n
SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY3)

2
rules/swss-common.mk
View File

@ -4,7 +4,7 @@ LIBSWSSCOMMON = libswsscommon_1.0.0_$(CONFIGURED_ARCH).deb
$(LIBSWSSCOMMON)_SRC_PATH = $(SRC_PATH)/sonic-swss-common
$(LIBSWSSCOMMON)_DEPENDS += $(LIBHIREDIS_DEV) $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \
$(LIBNL_ROUTE3_DEV) $(LIBNL_NF3_DEV) \
$(LIBNL_CLI_DEV) $(SWIG)
$(LIBNL_CLI_DEV) $(SWIG)
$(LIBSWSSCOMMON)_RDEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \
$(LIBNL_ROUTE3) $(LIBNL_NF3) $(LIBNL_CLI)
SONIC_DPKG_DEBS += $(LIBSWSSCOMMON)

1
rules/swsssdk-py2.mk
View File

@ -3,4 +3,5 @@
SWSSSDK_PY2 = swsssdk-2.0.1-py2-none-any.whl
$(SWSSSDK_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
$(SWSSSDK_PY2)_PYTHON_VERSION = 2
$(SWSSSDK_PY2)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
SONIC_PYTHON_WHEELS += $(SWSSSDK_PY2)

2
rules/swsssdk-py3.mk
View File

@ -4,5 +4,5 @@ SWSSSDK_PY3 = swsssdk-2.0.1-py3-none-any.whl
$(SWSSSDK_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
$(SWSSSDK_PY3)_PYTHON_VERSION = 3
# Synthetic dependency just to avoid race condition
$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2)
$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) $(REDIS_DUMP_LOAD_PY3)
SONIC_PYTHON_WHEELS += $(SWSSSDK_PY3)

1
slave.mk
View File

@ -816,6 +816,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))"
export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))"
export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"

9
src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch
View File

@ -33,7 +33,7 @@ index df2870b..1b6063b 100755
if pretty:
# hack to avoid implementing pretty printing
fp.write(dumps(host=host, port=port, password=password, db=db,
@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding):
@@ -276,28 +276,77 @@ def _read_key(key, r, pretty, encoding):
return (type, ttl, value)
def _reader(r, pretty, encoding, keys='*'):
@ -68,7 +68,7 @@ index df2870b..1b6063b 100755
+ i += 10000
+
+def _read_keys(r, encoded_keys, pretty, encoding):
+ decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys]
+ decoded_keys = [encoded_key.decode(encoding) for encoded_key in encoded_keys]
+ do_keys = decoded_keys
+ retries = 5
+ type_results = None
@ -84,12 +84,13 @@ index df2870b..1b6063b 100755
+ p = r.pipeline()
+ for key in do_keys:
+ p.type(key)
+ type_results = p.execute()
+ encoded_type_results = p.execute()
+ type_results = [encoded_type_result.decode('ascii') for encoded_type_result in encoded_type_results]
+
+ p = r.pipeline()
+ for i in range(len(do_keys)):
+ key = decoded_keys[i]
+ type = type_results[i].decode('ascii')
+ type = type_results[i]
+ if type == 'none':
+ # key was deleted by a concurrent operation on the data store.
+ # issue noops so that the number of results does not change