From a4d81f3c19cc4156a3d2b7d11cdb8a2e91f30107 Mon Sep 17 00:00:00 2001
From: Renuka Manavalan <47282725+renukamanavalan@users.noreply.github.com>
Date: Tue, 9 Mar 2021 19:49:54 -0800
Subject: [PATCH] Copy dummy flannel.conf to get around absence of CNI Network
 (#6985)

Why I did it
We skip install of CNI plugin, as we don't need. But this leaves node in "not ready" state, upon joining master.
To fix, we copy this dummy .conf file in /etc/cni/net.d

How I did it
Keep this file in /usr/share/sonic/templates and copy to /etc/cni/net.d upon joining k8s master.

How to verify it
Upon configuring master-IP and enable join, watch node join and move to ready state.
You may verify using kubectl get nodes command
---
 .../build_templates/sonic_debian_extension.j2 |  4 +++
 src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py           |  6 ++--
 src/sonic-ctrmgrd/ctrmgr/kube_commands.py     | 19 ++++++++-----
 src/sonic-ctrmgrd/tests/kube_commands_test.py | 28 ++++++++++++++-----
 4 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2
index 8edd9af847..9d2556edb5 100644
--- a/files/build_templates/sonic_debian_extension.j2
+++ b/files/build_templates/sonic_debian_extension.j2
@@ -403,6 +403,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3
 
 {% if include_kubernetes == "y" %}
+# Copy Flannel conf file into sonic-templates
+#
+sudo cp $BUILD_TEMPLATES/kube_cni.10-flannel.conflist $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
+
 # Install remote Container mgmt package
 # Required even if include_kubernetes != y, as it contains the
 # the container wrapper for docker start/stop/wait commands.
diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
index ba4f0057bd..49c11d2781 100755
--- a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
+++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
@@ -201,7 +201,7 @@ class MainServer:
         """ Modify entry for given table|key with given dict type data """
         conn = self.db_connectors[db_name]
         tbl = swsscommon.Table(conn, table_name)
-        print("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data)))
+        log_debug("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data)))
         tbl.set(key, list(data.items()))
 
 
@@ -242,7 +242,7 @@ class MainServer:
                 if not UNIT_TESTING:
                     raise Exception("Received error from select")
                 else:
-                    print("Skipped Exception; Received error from select")
+                    log_debug("Skipped Exception; Received error from select")
                     return
 
             for subscriber in self.subscribers:
@@ -588,7 +588,7 @@ def main():
     FeatureTransitionHandler(server)
     LabelsPendingHandler(server)
     server.run()
-    print("ctrmgrd.py main called")
+    log_debug("ctrmgrd.py main called")
     return 0
 
 
diff --git a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py
index 1ebfa606f0..db8cb348fc 100755
--- a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py
+++ b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py
@@ -22,6 +22,8 @@ KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
 KUBELET_YAML = "/var/lib/kubelet/config.yaml"
 SERVER_ADMIN_URL = "https://{}/admin.conf"
 LOCK_FILE = "/var/lock/kube_join.lock"
+FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
+CNI_DIR = "/etc/cni/net.d"
 
 # kubectl --kubeconfig <KUBE_ADMIN_CONF> label nodes
 #       <device_info.get_hostname()> <label to be added>
@@ -150,7 +152,7 @@ def func_get_labels(args):
         log_debug("Labels read failed.")
         return ret
 
-    print(json.dumps(node_labels, indent=4))
+    log_debug(json.dumps(node_labels, indent=4))
     return 0
     
 
@@ -174,7 +176,7 @@ def is_connected(server=""):
 def func_is_connected(args):
     """ Get connected state """
     connected = is_connected()
-    print("Currently {} to Kube master".format(
+    log_debug("Currently {} to Kube master".format(
         "connected" if connected else "not connected"))
     return 0 if connected else 1
 
@@ -212,11 +214,11 @@ def _download_file(server, port, insecure):
             str(port), server, str(port), fname, update_file)
     (ret, _, err) = _run_command(cmd)
 
-    print("sed command: ret={}".format(ret))
+    log_debug("sed command: ret={}".format(ret))
     if ret != 0:
         log_error("sed update of downloaded file failed with ret={}".
                 format(ret))
-        print("sed command failed: ret={}".format(ret))
+        log_debug("sed command failed: ret={}".format(ret))
         return ret
 
     shutil.copyfile(update_file, KUBE_ADMIN_CONF)
@@ -272,7 +274,7 @@ def _do_reset(pending_join = False):
                 format(KUBE_ADMIN_CONF, device_info.get_hostname()))
 
     _run_command("kubeadm reset -f", 10)
-    _run_command("rm -rf /etc/cni/net.d")
+    _run_command("rm -rf {}".format(CNI_DIR))
     if not pending_join:
         _run_command("rm -f {}".format(KUBE_ADMIN_CONF))
     _run_command("systemctl stop kubelet")
@@ -284,16 +286,19 @@ def _do_join(server, port, insecure):
     out = ""
     try:
         ret = _download_file(server, port, insecure)
-        print("_download ret={}".format(ret))
+        log_debug("_download ret={}".format(ret))
         if ret == 0:
             _do_reset(True)
             _run_command("modprobe br_netfilter")
+            # Copy flannel.conf
+            _run_command("mkdir -p {}".format(CNI_DIR))
+            _run_command("cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR))
             (ret, _, _) = _run_command("systemctl start kubelet")
 
         if ret == 0:
             (ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
                 KUBE_ADMIN_CONF, device_info.get_hostname()), timeout=60)
-            print("ret = {}".format(ret))
+            log_debug("ret = {}".format(ret))
 
     except IOError as e:
         err = "Download failed: {}".format(str(e))
diff --git a/src/sonic-ctrmgrd/tests/kube_commands_test.py b/src/sonic-ctrmgrd/tests/kube_commands_test.py
index 84260d3488..b76c23e100 100755
--- a/src/sonic-ctrmgrd/tests/kube_commands_test.py
+++ b/src/sonic-ctrmgrd/tests/kube_commands_test.py
@@ -13,6 +13,8 @@ import kube_commands
 
 
 KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
+FLANNEL_CONF_FILE = "/tmp/flannel.conf"
+CNI_DIR = "/tmp/cni/net.d"
 
 # kube_commands test cases
 # NOTE: Ensure state-db entry is complete in PRE as we need to
@@ -108,9 +110,11 @@ join_test_data = {
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "systemctl stop kubelet",
             "modprobe br_netfilter",
+            "mkdir -p {}".format(CNI_DIR),
+            "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
             "kubeadm join --discovery-file {} --node-name None".format(
                 KUBE_ADMIN_CONF)
@@ -129,9 +133,11 @@ None".format(KUBE_ADMIN_CONF),
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "systemctl stop kubelet",
             "modprobe br_netfilter",
+            "mkdir -p {}".format(CNI_DIR),
+            "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
             "kubeadm join --discovery-file {} --node-name None".format(
                 KUBE_ADMIN_CONF)
@@ -159,9 +165,11 @@ None".format(KUBE_ADMIN_CONF),
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "systemctl stop kubelet",
             "modprobe br_netfilter",
+            "mkdir -p {}".format(CNI_DIR),
+            "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
             "kubeadm join --discovery-file {} --node-name None".format(
                 KUBE_ADMIN_CONF)
@@ -181,9 +189,11 @@ None".format(KUBE_ADMIN_CONF),
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "systemctl stop kubelet",
             "modprobe br_netfilter",
+            "mkdir -p {}".format(CNI_DIR),
+            "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
             "kubeadm join --discovery-file {} --node-name None".format(
                 KUBE_ADMIN_CONF)
@@ -213,7 +223,7 @@ reset_test_data = {
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "rm -f {}".format(KUBE_ADMIN_CONF),
             "systemctl stop kubelet"
         ]
@@ -228,7 +238,7 @@ None".format(KUBE_ADMIN_CONF),
             "kubectl --kubeconfig {} --request-timeout 20s delete node \
 None".format(KUBE_ADMIN_CONF),
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "rm -f {}".format(KUBE_ADMIN_CONF),
             "systemctl stop kubelet"
         ]
@@ -239,7 +249,7 @@ None".format(KUBE_ADMIN_CONF),
         common_test.ARGS: [True],
         common_test.PROC_CMD: [
             "kubeadm reset -f",
-            "rm -rf /etc/cni/net.d",
+            "rm -rf {}".format(CNI_DIR),
             "rm -f {}".format(KUBE_ADMIN_CONF),
             "systemctl stop kubelet"
         ]
@@ -269,7 +279,11 @@ clusters:\n\
         kubelet_yaml = "/tmp/kubelet_config.yaml"
         with open(kubelet_yaml, "w") as s:
             s.close()
+        with open(FLANNEL_CONF_FILE, "w") as s:
+            s.close()
         kube_commands.KUBELET_YAML = kubelet_yaml
+        kube_commands.CNI_DIR = CNI_DIR
+        kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
         kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
         kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF