[MultiDB] add mutidb warmboot support - restoring database (#5773)

* restoring each database  with all data before warmboot and then flush unused data in each instance, following the multiDB warmboot design at https://github.com/Azure/SONiC/blob/master/doc/database/multi_database_instances.md
  * restore needs to be done in database docker since we need to know the database_config.json in new version
  * copy all data rdb file into each instance restoration location andthen flush unused database
  * other logic is the same as before
*  backing up database part is in another PR at sonic-utilities https://github.com/Azure/sonic-utilities/pull/1205, they depend on each other

dockers/docker-database/Dockerfile.j2

@@ -58,5 +58,6 @@ COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["files/sysctl-net.conf", "/etc/sysctl.d/"]
 COPY ["critical_processes", "/etc/supervisor"]
 COPY ["files/update_chassisdb_config", "/usr/local/bin/"]
+COPY ["flush_unused_database", "/usr/local/bin/"]
 
 ENTRYPOINT ["/usr/local/bin/docker-database-init.sh"]

dockers/docker-database/docker-database-init.sh

@@ -77,4 +77,20 @@ else
 fi
 rm $db_cfg_file_tmp
 
+# copy dump.rdb file to each instance for restoration
+DUMPFILE=/var/lib/redis/dump.rdb
+redis_inst_list=`/usr/bin/python -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"`
+for inst in $redis_inst_list
+do
+    mkdir -p /var/lib/$inst
+    if [[ -f $DUMPFILE ]]; then
+        # copy warmboot rdb file into each new instance location
+        if [[ "$DUMPFILE" != "/var/lib/$inst/dump.rdb" ]]; then
+            cp $DUMPFILE /var/lib/$inst/dump.rdb
+        fi
+    else
+        echo -n > /var/lib/$inst/dump.rdb
+    fi
+done
+
 exec /usr/local/bin/supervisord

dockers/docker-database/flush_unused_database

@@ -0,0 +1,28 @@
+#!/usr/bin/python
+import swsssdk
+import redis
+import subprocess
+import time
+
+while(True):
+    output = subprocess.Popen(['sonic-db-cli', 'PING'], stdout=subprocess.PIPE).communicate()[0]
+    if 'PONG' in output:
+        break
+    time.sleep(1)
+
+instlists = swsssdk.SonicDBConfig.get_instancelist()
+for instname, v in instlists.items():
+    insthost = v['hostname']
+    instsocket = v['unix_socket_path']
+
+    dblists = swsssdk.SonicDBConfig.get_dblist()
+    for dbname in dblists:
+        dbid = swsssdk.SonicDBConfig.get_dbid(dbname)
+        dbinst = swsssdk.SonicDBConfig.get_instancename(dbname)
+
+        # this DB is on current instance, skip flush
+        if dbinst == instname:
+            continue
+
+        r = redis.Redis(host=insthost, unix_socket_path=instsocket, db=dbid)
+        r.flushdb()

dockers/docker-database/supervisord.conf.j2

@@ -33,3 +33,11 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 {%     endfor %}
 {% endif %}
+
+[program:flushdb]
+command=/bin/bash -c "sleep 300 && /usr/local/bin/flush_unused_database"
+priority=3
+autostart=true
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog