From 465ebbafffe3f3deec62e8eaed73e5bdd18089b6 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Tue, 20 Nov 2018 19:27:56 -0800 Subject: [PATCH] Build patched redis-dump-load (#2277) * Build patched redis-dump-load * Fix build * Add build rule --- .gitmodules | 3 + .../build_templates/sonic_debian_extension.j2 | 13 +- rules/redis-dump-load-py2.mk | 6 + slave.mk | 4 +- src/redis-dump-load | 1 + .../0001-Use-pipelines-when-dumping-52.patch | 154 ++++++++++++++++++ ...ix-setup.py-for-test-and-bdist_wheel.patch | 26 +++ src/redis-dump-load.patch/series | 2 + 8 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 rules/redis-dump-load-py2.mk create mode 160000 src/redis-dump-load create mode 100644 src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch create mode 100644 src/redis-dump-load.patch/0002-Fix-setup.py-for-test-and-bdist_wheel.patch create mode 100644 src/redis-dump-load.patch/series diff --git a/.gitmodules b/.gitmodules index 79905318f8..9adb11eeb0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -62,3 +62,6 @@ [submodule "platform/mellanox/hw-management/hw-mgmt"] path = platform/mellanox/hw-management/hw-mgmt url = https://github.com/Mellanox/hw-mgmt/ +[submodule "src/redis-dump-load"] + path = src/redis-dump-load + url = https://github.com/p/redis-dump-load.git diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 387ab563cf..d215626725 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -63,7 +63,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/ sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/ sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ -# Install dependencies for SONiC config engine +# Install dependencies for SONiC config engine sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \ python-dev \ python-lxml \ @@ -78,7 +78,12 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME # Install Python client for Redis sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6" -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install redis-dump-load + +# Install redis-dump-load Python 2 package +REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}}) +sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $REDIS_DUMP_LOAD_PY2_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME # Install SwSS SDK Python 2 package SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}}) @@ -292,9 +297,9 @@ sudo LANG=C cp $SCRIPTS_DIR/syncd.sh $FILESYSTEM_ROOT/usr/local/bin/syncd.sh sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/ sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer -sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev +sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y -sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y +sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y {% for file in installer_extra_files.split(' ') -%} {% if file.strip() -%} diff --git a/rules/redis-dump-load-py2.mk b/rules/redis-dump-load-py2.mk new file mode 100644 index 0000000000..5f756caada --- /dev/null +++ b/rules/redis-dump-load-py2.mk @@ -0,0 +1,6 @@ +# redis_dump_load python2 wheel + +REDIS_DUMP_LOAD_PY2 = redis_dump_load-1.1-py2-none-any.whl +$(REDIS_DUMP_LOAD_PY2)_SRC_PATH = $(SRC_PATH)/redis-dump-load +$(REDIS_DUMP_LOAD_PY2)_PYTHON_VERSION = 2 +SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY2) diff --git a/slave.mk b/slave.mk index e6306d1661..a2bd053113 100644 --- a/slave.mk +++ b/slave.mk @@ -479,7 +479,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(LIBNSS_TACPLUS)) \ $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \ $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \ - $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) + $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \ + $$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) $(HEADER) # Pass initramfs and linux kernel explicitly. They are used for all platforms export initramfs_tools="$(DEBS_PATH)/$(INITRAMFS_TOOLS)" @@ -498,6 +499,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))" export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))" export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))" + export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))" $(foreach docker, $($*_DOCKERS),\ export docker_image="$(docker)" diff --git a/src/redis-dump-load b/src/redis-dump-load new file mode 160000 index 0000000000..832a645e4d --- /dev/null +++ b/src/redis-dump-load @@ -0,0 +1 @@ +Subproject commit 832a645e4ddff0f38ec0d64e3be70f48013287e6 diff --git a/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch b/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch new file mode 100644 index 0000000000..5eea3faba2 --- /dev/null +++ b/src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch @@ -0,0 +1,154 @@ +From ed20dced07d8b2d140e2c1d79d506be0e12f339e Mon Sep 17 00:00:00 2001 +From: Oleg Pudeyev +Date: Sat, 28 Jan 2017 15:37:43 -0500 +Subject: [PATCH] Use pipelines when dumping, #52 + +--- + redisdl.py | 102 +++++++++++++++++++++++++++++++++++++++-------------- + 1 file changed, 75 insertions(+), 27 deletions(-) + +diff --git a/redisdl.py b/redisdl.py +index df2870b..1b6063b 100755 +--- a/redisdl.py ++++ b/redisdl.py +@@ -141,18 +141,18 @@ def dumps(host='localhost', port=6379, password=None, db=0, pretty=False, + class BytesWriteWrapper(object): + def __init__(self, stream): + self.stream = stream +- ++ + def write(self, str): + return self.stream.write(str.encode()) + + def dump(fp, host='localhost', port=6379, password=None, db=0, pretty=False, + unix_socket_path=None, encoding='utf-8', keys='*'): +- ++ + try: + fp.write('') + except TypeError: + fp = BytesWriteWrapper(fp) +- ++ + if pretty: + # hack to avoid implementing pretty printing + fp.write(dumps(host=host, port=port, password=password, db=db, +@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding): + return (type, ttl, value) + + def _reader(r, pretty, encoding, keys='*'): +- for encoded_key in r.keys(keys): +- key = encoded_key.decode(encoding) +- handled = False +- for i in range(10): +- try: +- type, ttl, value = _read_key(encoded_key, r, pretty, encoding) +- yield key, type, ttl, value +- handled = True +- break +- except KeyDeletedError: +- # do not dump the key +- handled = True +- break +- except redis.WatchError: +- # same logic as key type changed +- pass +- except KeyTypeChangedError: +- # retry reading type again +- pass +- if not handled: +- # ran out of retries +- raise ConcurrentModificationError('Key %s is being concurrently modified' % key) ++ encoded_keys = r.keys(keys) ++ i = 0 ++ while i < len(encoded_keys): ++ for key, type, ttl, value in _read_keys(r, encoded_keys[i:i+10000], ++ pretty=pretty, encoding=encoding): ++ yield key, type, ttl, value ++ i += 10000 ++ ++def _read_keys(r, encoded_keys, pretty, encoding): ++ decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys] ++ do_keys = decoded_keys ++ retries = 5 ++ type_results = None ++ while len(do_keys) > 0 and retries > 0: ++ next_do_keys = [] ++ next_type_results = [] ++ ++ if type_results is None: ++ # first pass, need to get the types. ++ # on subsequent passes we know the types ++ # because the previous pass retrieved them and ++ # found a type mismatch ++ p = r.pipeline() ++ for key in do_keys: ++ p.type(key) ++ type_results = p.execute() ++ ++ p = r.pipeline() ++ for i in range(len(do_keys)): ++ key = decoded_keys[i] ++ type = type_results[i].decode('ascii') ++ if type == 'none': ++ # key was deleted by a concurrent operation on the data store. ++ # issue noops so that the number of results does not change ++ p.type(key) ++ p.type(key) ++ p.type(key) ++ continue ++ reader = readers.get(type) ++ if reader is None: ++ raise UnknownTypeError("Unknown key type: %s" % type) ++ reader.send_command(p, key) ++ r.pttl_or_ttl_pipeline(p, key) ++ p.type(key) ++ results = p.execute() ++ ++ for i in range(len(do_keys)): ++ key = decoded_keys[i] ++ original_type = type_results[i] ++ if original_type == 'none': ++ # this is where we actually skip a key that was deleted ++ # by concurrent operations ++ continue ++ final_type = results[i*3+2].decode('ascii') ++ if original_type != final_type: ++ # type changed, will retry ++ next_do_keys.append(key) ++ # need to update expected type ++ next_type_results.append(final_type) ++ continue ++ reader = readers.get(original_type) ++ value = reader.handle_response(results[i*3], pretty, encoding) ++ ttl = r.decode_pttl_or_ttl_pipeline_value(results[i*3+1]) ++ yield key, final_type, ttl, value ++ retries -= 1 ++ do_keys = next_do_keys ++ type_results = next_type_results ++ ++ if len(do_keys) > 0: ++ raise ConcurrentModificationError('Keys %s are being concurrently modified' % ', '.join(do_keys)) + + def _empty(r): + for key in r.keys(): +@@ -372,14 +420,14 @@ def ijson_top_level_items(file, local_streaming_backend): + class TextReadWrapper(object): + def __init__(self, fp): + self.fp = fp +- ++ + def read(self, *args, **kwargs): + return self.fp.read(*args, **kwargs).decode() + + class BytesReadWrapper(object): + def __init__(self, fp): + self.fp = fp +- ++ + def read(self, *args, **kwargs): + return self.fp.read(*args, **kwargs).encode('utf-8') + +-- +2.18.0 + diff --git a/src/redis-dump-load.patch/0002-Fix-setup.py-for-test-and-bdist_wheel.patch b/src/redis-dump-load.patch/0002-Fix-setup.py-for-test-and-bdist_wheel.patch new file mode 100644 index 0000000000..5bb319a617 --- /dev/null +++ b/src/redis-dump-load.patch/0002-Fix-setup.py-for-test-and-bdist_wheel.patch @@ -0,0 +1,26 @@ +From c2c93fa3b702a4f2364383fd4ae69763068686d2 Mon Sep 17 00:00:00 2001 +From: Qi Luo +Date: Tue, 20 Nov 2018 03:21:31 +0000 +Subject: [PATCH] Fix setup.py for test and bdist_wheel + +Signed-off-by: Qi Luo +--- + setup.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/setup.py b/setup.py +index 8ccf31f..6457163 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,7 +1,7 @@ + #!/usr/bin/env python + + import os.path +-from distutils.core import setup ++from setuptools import setup, find_packages + + package_name = 'redis-dump-load' + package_version = '1.1' +-- +2.18.0 + diff --git a/src/redis-dump-load.patch/series b/src/redis-dump-load.patch/series new file mode 100644 index 0000000000..c73c3b5caa --- /dev/null +++ b/src/redis-dump-load.patch/series @@ -0,0 +1,2 @@ +0001-Use-pipelines-when-dumping-52.patch +0002-Fix-setup.py-for-test-and-bdist_wheel.patch