[baseimage]: Build and install redis-dump-load Python 3 package in host image (#4661)

Fix #4656
This commit is contained in:
Qi Luo 2020-05-30 05:52:27 -07:00 committed by GitHub
parent d86af98930
commit 65e7a84509
No account linked to committer's email address
7 changed files with 32 additions and 12 deletions

View File

@ -101,6 +101,13 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
# Install Python client for Redis # Install Python client for Redis
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6" sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"
# Install redis-dump-load Python 3 package
# Note: the scripts will be overwritten by corresponding Python 2 package
REDIS_DUMP_LOAD_PY3_WHEEL_NAME=$(basename {{redis_dump_load_py3_wheel_path}})
sudo cp {{redis_dump_load_py3_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $REDIS_DUMP_LOAD_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
# Install redis-dump-load Python 2 package # Install redis-dump-load Python 2 package
REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}}) REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
@ -110,13 +117,8 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
# Install Python module for ipaddress # Install Python module for ipaddress
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress
# Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
# Install SwSS SDK Python 3 package # Install SwSS SDK Python 3 package
# Note: the scripts will be overwritten by corresponding Python 2 package
if [ -e {{swsssdk_py3_wheel_path}} ]; then if [ -e {{swsssdk_py3_wheel_path}} ]; then
SWSSSDK_PY3_WHEEL_NAME=$(basename {{swsssdk_py3_wheel_path}}) SWSSSDK_PY3_WHEEL_NAME=$(basename {{swsssdk_py3_wheel_path}})
sudo cp {{swsssdk_py3_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME sudo cp {{swsssdk_py3_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
@ -124,6 +126,12 @@ if [ -e {{swsssdk_py3_wheel_path}} ]; then
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
fi fi
# Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
# Install sonic-yang-models py3 package, install dependencies # Install sonic-yang-models py3 package, install dependencies
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb

View File

@ -0,0 +1,9 @@
# redis_dump_load python3 wheel
REDIS_DUMP_LOAD_PY3 = redis_dump_load-1.1-py3-none-any.whl
$(REDIS_DUMP_LOAD_PY3)_SRC_PATH = $(SRC_PATH)/redis-dump-load
$(REDIS_DUMP_LOAD_PY3)_PYTHON_VERSION = 3
# Synthetic dependency just to avoid race condition
$(REDIS_DUMP_LOAD_PY3)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
$(REDIS_DUMP_LOAD_PY3)_TEST = n
SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY3)

View File

@ -3,4 +3,5 @@
SWSSSDK_PY2 = swsssdk-2.0.1-py2-none-any.whl SWSSSDK_PY2 = swsssdk-2.0.1-py2-none-any.whl
$(SWSSSDK_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk $(SWSSSDK_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
$(SWSSSDK_PY2)_PYTHON_VERSION = 2 $(SWSSSDK_PY2)_PYTHON_VERSION = 2
$(SWSSSDK_PY2)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
SONIC_PYTHON_WHEELS += $(SWSSSDK_PY2) SONIC_PYTHON_WHEELS += $(SWSSSDK_PY2)

View File

@ -4,5 +4,5 @@ SWSSSDK_PY3 = swsssdk-2.0.1-py3-none-any.whl
$(SWSSSDK_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk $(SWSSSDK_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
$(SWSSSDK_PY3)_PYTHON_VERSION = 3 $(SWSSSDK_PY3)_PYTHON_VERSION = 3
# Synthetic dependency just to avoid race condition # Synthetic dependency just to avoid race condition
$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) $(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) $(REDIS_DUMP_LOAD_PY3)
SONIC_PYTHON_WHEELS += $(SWSSSDK_PY3) SONIC_PYTHON_WHEELS += $(SWSSSDK_PY3)

View File

@ -816,6 +816,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))" export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))"
export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))" export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))" export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))"
export install_debug_image="$(INSTALL_DEBUG_TOOLS)" export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))" export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))" export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"

View File

@ -33,7 +33,7 @@ index df2870b..1b6063b 100755
if pretty: if pretty:
# hack to avoid implementing pretty printing # hack to avoid implementing pretty printing
fp.write(dumps(host=host, port=port, password=password, db=db, fp.write(dumps(host=host, port=port, password=password, db=db,
@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding): @@ -276,28 +276,77 @@ def _read_key(key, r, pretty, encoding):
return (type, ttl, value) return (type, ttl, value)
def _reader(r, pretty, encoding, keys='*'): def _reader(r, pretty, encoding, keys='*'):
@ -68,7 +68,7 @@ index df2870b..1b6063b 100755
+ i += 10000 + i += 10000
+ +
+def _read_keys(r, encoded_keys, pretty, encoding): +def _read_keys(r, encoded_keys, pretty, encoding):
+ decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys] + decoded_keys = [encoded_key.decode(encoding) for encoded_key in encoded_keys]
+ do_keys = decoded_keys + do_keys = decoded_keys
+ retries = 5 + retries = 5
+ type_results = None + type_results = None
@ -84,12 +84,13 @@ index df2870b..1b6063b 100755
+ p = r.pipeline() + p = r.pipeline()
+ for key in do_keys: + for key in do_keys:
+ p.type(key) + p.type(key)
+ type_results = p.execute() + encoded_type_results = p.execute()
+ type_results = [encoded_type_result.decode('ascii') for encoded_type_result in encoded_type_results]
+ +
+ p = r.pipeline() + p = r.pipeline()
+ for i in range(len(do_keys)): + for i in range(len(do_keys)):
+ key = decoded_keys[i] + key = decoded_keys[i]
+ type = type_results[i].decode('ascii') + type = type_results[i]
+ if type == 'none': + if type == 'none':
+ # key was deleted by a concurrent operation on the data store. + # key was deleted by a concurrent operation on the data store.
+ # issue noops so that the number of results does not change + # issue noops so that the number of results does not change