Build patched redis-dump-load (#2277)

* Build patched redis-dump-load
* Fix build
* Add build rule
This commit is contained in:
Qi Luo 2018-11-20 19:27:56 -08:00 committed by GitHub
parent 0f6c29e465
commit 465ebbafff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 204 additions and 5 deletions

3
.gitmodules vendored
View File

@ -62,3 +62,6 @@
[submodule "platform/mellanox/hw-management/hw-mgmt"] [submodule "platform/mellanox/hw-management/hw-mgmt"]
path = platform/mellanox/hw-management/hw-mgmt path = platform/mellanox/hw-management/hw-mgmt
url = https://github.com/Mellanox/hw-mgmt/ url = https://github.com/Mellanox/hw-mgmt/
[submodule "src/redis-dump-load"]
path = src/redis-dump-load
url = https://github.com/p/redis-dump-load.git

View File

@ -63,7 +63,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/
sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/ sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/
sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
# Install dependencies for SONiC config engine # Install dependencies for SONiC config engine
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
python-dev \ python-dev \
python-lxml \ python-lxml \
@ -78,7 +78,12 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
# Install Python client for Redis # Install Python client for Redis
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6" sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install redis-dump-load
# Install redis-dump-load Python 2 package
REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $REDIS_DUMP_LOAD_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
# Install SwSS SDK Python 2 package # Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}}) SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
@ -292,9 +297,9 @@ sudo LANG=C cp $SCRIPTS_DIR/syncd.sh $FILESYSTEM_ROOT/usr/local/bin/syncd.sh
sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/ sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get remove -y python-dev
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
{% for file in installer_extra_files.split(' ') -%} {% for file in installer_extra_files.split(' ') -%}
{% if file.strip() -%} {% if file.strip() -%}

View File

@ -0,0 +1,6 @@
# redis_dump_load python2 wheel
REDIS_DUMP_LOAD_PY2 = redis_dump_load-1.1-py2-none-any.whl
$(REDIS_DUMP_LOAD_PY2)_SRC_PATH = $(SRC_PATH)/redis-dump-load
$(REDIS_DUMP_LOAD_PY2)_PYTHON_VERSION = 2
SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY2)

View File

@ -479,7 +479,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(LIBNSS_TACPLUS)) \ $(LIBNSS_TACPLUS)) \
$$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \ $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
$$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \ $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \
$$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) $$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \
$$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))
$(HEADER) $(HEADER)
# Pass initramfs and linux kernel explicitly. They are used for all platforms # Pass initramfs and linux kernel explicitly. They are used for all platforms
export initramfs_tools="$(DEBS_PATH)/$(INITRAMFS_TOOLS)" export initramfs_tools="$(DEBS_PATH)/$(INITRAMFS_TOOLS)"
@ -498,6 +499,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))" export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))"
export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))" export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))"
export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))" export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
$(foreach docker, $($*_DOCKERS),\ $(foreach docker, $($*_DOCKERS),\
export docker_image="$(docker)" export docker_image="$(docker)"

1
src/redis-dump-load Submodule

@ -0,0 +1 @@
Subproject commit 832a645e4ddff0f38ec0d64e3be70f48013287e6

View File

@ -0,0 +1,154 @@
From ed20dced07d8b2d140e2c1d79d506be0e12f339e Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sat, 28 Jan 2017 15:37:43 -0500
Subject: [PATCH] Use pipelines when dumping, #52
---
redisdl.py | 102 +++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 75 insertions(+), 27 deletions(-)
diff --git a/redisdl.py b/redisdl.py
index df2870b..1b6063b 100755
--- a/redisdl.py
+++ b/redisdl.py
@@ -141,18 +141,18 @@ def dumps(host='localhost', port=6379, password=None, db=0, pretty=False,
class BytesWriteWrapper(object):
def __init__(self, stream):
self.stream = stream
-
+
def write(self, str):
return self.stream.write(str.encode())
def dump(fp, host='localhost', port=6379, password=None, db=0, pretty=False,
unix_socket_path=None, encoding='utf-8', keys='*'):
-
+
try:
fp.write('')
except TypeError:
fp = BytesWriteWrapper(fp)
-
+
if pretty:
# hack to avoid implementing pretty printing
fp.write(dumps(host=host, port=port, password=password, db=db,
@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding):
return (type, ttl, value)
def _reader(r, pretty, encoding, keys='*'):
- for encoded_key in r.keys(keys):
- key = encoded_key.decode(encoding)
- handled = False
- for i in range(10):
- try:
- type, ttl, value = _read_key(encoded_key, r, pretty, encoding)
- yield key, type, ttl, value
- handled = True
- break
- except KeyDeletedError:
- # do not dump the key
- handled = True
- break
- except redis.WatchError:
- # same logic as key type changed
- pass
- except KeyTypeChangedError:
- # retry reading type again
- pass
- if not handled:
- # ran out of retries
- raise ConcurrentModificationError('Key %s is being concurrently modified' % key)
+ encoded_keys = r.keys(keys)
+ i = 0
+ while i < len(encoded_keys):
+ for key, type, ttl, value in _read_keys(r, encoded_keys[i:i+10000],
+ pretty=pretty, encoding=encoding):
+ yield key, type, ttl, value
+ i += 10000
+
+def _read_keys(r, encoded_keys, pretty, encoding):
+ decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys]
+ do_keys = decoded_keys
+ retries = 5
+ type_results = None
+ while len(do_keys) > 0 and retries > 0:
+ next_do_keys = []
+ next_type_results = []
+
+ if type_results is None:
+ # first pass, need to get the types.
+ # on subsequent passes we know the types
+ # because the previous pass retrieved them and
+ # found a type mismatch
+ p = r.pipeline()
+ for key in do_keys:
+ p.type(key)
+ type_results = p.execute()
+
+ p = r.pipeline()
+ for i in range(len(do_keys)):
+ key = decoded_keys[i]
+ type = type_results[i].decode('ascii')
+ if type == 'none':
+ # key was deleted by a concurrent operation on the data store.
+ # issue noops so that the number of results does not change
+ p.type(key)
+ p.type(key)
+ p.type(key)
+ continue
+ reader = readers.get(type)
+ if reader is None:
+ raise UnknownTypeError("Unknown key type: %s" % type)
+ reader.send_command(p, key)
+ r.pttl_or_ttl_pipeline(p, key)
+ p.type(key)
+ results = p.execute()
+
+ for i in range(len(do_keys)):
+ key = decoded_keys[i]
+ original_type = type_results[i]
+ if original_type == 'none':
+ # this is where we actually skip a key that was deleted
+ # by concurrent operations
+ continue
+ final_type = results[i*3+2].decode('ascii')
+ if original_type != final_type:
+ # type changed, will retry
+ next_do_keys.append(key)
+ # need to update expected type
+ next_type_results.append(final_type)
+ continue
+ reader = readers.get(original_type)
+ value = reader.handle_response(results[i*3], pretty, encoding)
+ ttl = r.decode_pttl_or_ttl_pipeline_value(results[i*3+1])
+ yield key, final_type, ttl, value
+ retries -= 1
+ do_keys = next_do_keys
+ type_results = next_type_results
+
+ if len(do_keys) > 0:
+ raise ConcurrentModificationError('Keys %s are being concurrently modified' % ', '.join(do_keys))
def _empty(r):
for key in r.keys():
@@ -372,14 +420,14 @@ def ijson_top_level_items(file, local_streaming_backend):
class TextReadWrapper(object):
def __init__(self, fp):
self.fp = fp
-
+
def read(self, *args, **kwargs):
return self.fp.read(*args, **kwargs).decode()
class BytesReadWrapper(object):
def __init__(self, fp):
self.fp = fp
-
+
def read(self, *args, **kwargs):
return self.fp.read(*args, **kwargs).encode('utf-8')
--
2.18.0

View File

@ -0,0 +1,26 @@
From c2c93fa3b702a4f2364383fd4ae69763068686d2 Mon Sep 17 00:00:00 2001
From: Qi Luo <qiluo-msft@users.noreply.github.com>
Date: Tue, 20 Nov 2018 03:21:31 +0000
Subject: [PATCH] Fix setup.py for test and bdist_wheel
Signed-off-by: Qi Luo <qiluo-msft@users.noreply.github.com>
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 8ccf31f..6457163 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
import os.path
-from distutils.core import setup
+from setuptools import setup, find_packages
package_name = 'redis-dump-load'
package_version = '1.1'
--
2.18.0

View File

@ -0,0 +1,2 @@
0001-Use-pipelines-when-dumping-52.patch
0002-Fix-setup.py-for-test-and-bdist_wheel.patch