matthew/sonic-buildimage
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Copy dummy flannel.conf to get around absence of CNI Network (#6985)

Browse Source 
Why I did it
We skip install of CNI plugin, as we don't need. But this leaves node in "not ready" state, upon joining master.
To fix, we copy this dummy .conf file in /etc/cni/net.d

How I did it
Keep this file in /usr/share/sonic/templates and copy to /etc/cni/net.d upon joining k8s master.

How to verify it
Upon configuring master-IP and enable join, watch node join and move to ready state.
You may verify using kubectl get nodes command
This commit is contained in:
Renuka Manavalan 2021-03-09 19:49:54 -08:00 committed by Guohan Lu
parent 97426aff5a
commit a4d81f3c19
4 changed files with 40 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
files/build_templates/sonic_debian_extension.j2
View File

@ -403,6 +403,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3
{% if include_kubernetes == "y" %} {% if include_kubernetes == "y" %}
# Copy Flannel conf file into sonic-templates
#
sudo cp $BUILD_TEMPLATES/kube_cni.10-flannel.conflist $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
# Install remote Container mgmt package # Install remote Container mgmt package
# Required even if include_kubernetes != y, as it contains the # Required even if include_kubernetes != y, as it contains the
# the container wrapper for docker start/stop/wait commands. # the container wrapper for docker start/stop/wait commands.

6
src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
View File

@ -201,7 +201,7 @@ class MainServer:
""" Modify entry for given table|key with given dict type data """ """ Modify entry for given table|key with given dict type data """
conn = self.db_connectors[db_name] conn = self.db_connectors[db_name]
tbl = swsscommon.Table(conn, table_name) tbl = swsscommon.Table(conn, table_name)
print("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data))) log_debug("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data)))
tbl.set(key, list(data.items())) tbl.set(key, list(data.items()))
@ -242,7 +242,7 @@ class MainServer:
if not UNIT_TESTING: if not UNIT_TESTING:
raise Exception("Received error from select") raise Exception("Received error from select")
else: else:
print("Skipped Exception; Received error from select") log_debug("Skipped Exception; Received error from select")
return return
for subscriber in self.subscribers: for subscriber in self.subscribers:
@ -588,7 +588,7 @@ def main():
FeatureTransitionHandler(server) FeatureTransitionHandler(server)
LabelsPendingHandler(server) LabelsPendingHandler(server)
server.run() server.run()
print("ctrmgrd.py main called") log_debug("ctrmgrd.py main called")
return 0 return 0

19
src/sonic-ctrmgrd/ctrmgr/kube_commands.py
View File

@ -22,6 +22,8 @@ KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
KUBELET_YAML = "/var/lib/kubelet/config.yaml" KUBELET_YAML = "/var/lib/kubelet/config.yaml"
SERVER_ADMIN_URL = "https://{}/admin.conf" SERVER_ADMIN_URL = "https://{}/admin.conf"
LOCK_FILE = "/var/lock/kube_join.lock" LOCK_FILE = "/var/lock/kube_join.lock"
FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
CNI_DIR = "/etc/cni/net.d"
# kubectl --kubeconfig <KUBE_ADMIN_CONF> label nodes # kubectl --kubeconfig <KUBE_ADMIN_CONF> label nodes
# <device_info.get_hostname()> <label to be added> # <device_info.get_hostname()> <label to be added>
@ -150,7 +152,7 @@ def func_get_labels(args):
log_debug("Labels read failed.") log_debug("Labels read failed.")
return ret return ret
print(json.dumps(node_labels, indent=4)) log_debug(json.dumps(node_labels, indent=4))
return 0 return 0
@ -174,7 +176,7 @@ def is_connected(server=""):
def func_is_connected(args): def func_is_connected(args):
""" Get connected state """ """ Get connected state """
connected = is_connected() connected = is_connected()
print("Currently {} to Kube master".format( log_debug("Currently {} to Kube master".format(
"connected" if connected else "not connected")) "connected" if connected else "not connected"))
return 0 if connected else 1 return 0 if connected else 1
@ -212,11 +214,11 @@ def _download_file(server, port, insecure):
str(port), server, str(port), fname, update_file) str(port), server, str(port), fname, update_file)
(ret, _, err) = _run_command(cmd) (ret, _, err) = _run_command(cmd)
print("sed command: ret={}".format(ret)) log_debug("sed command: ret={}".format(ret))
if ret != 0: if ret != 0:
log_error("sed update of downloaded file failed with ret={}". log_error("sed update of downloaded file failed with ret={}".
format(ret)) format(ret))
print("sed command failed: ret={}".format(ret)) log_debug("sed command failed: ret={}".format(ret))
return ret return ret
shutil.copyfile(update_file, KUBE_ADMIN_CONF) shutil.copyfile(update_file, KUBE_ADMIN_CONF)
@ -272,7 +274,7 @@ def _do_reset(pending_join = False):
format(KUBE_ADMIN_CONF, device_info.get_hostname())) format(KUBE_ADMIN_CONF, device_info.get_hostname()))
_run_command("kubeadm reset -f", 10) _run_command("kubeadm reset -f", 10)
_run_command("rm -rf /etc/cni/net.d") _run_command("rm -rf {}".format(CNI_DIR))
if not pending_join: if not pending_join:
_run_command("rm -f {}".format(KUBE_ADMIN_CONF)) _run_command("rm -f {}".format(KUBE_ADMIN_CONF))
_run_command("systemctl stop kubelet") _run_command("systemctl stop kubelet")
@ -284,16 +286,19 @@ def _do_join(server, port, insecure):
out = "" out = ""
try: try:
ret = _download_file(server, port, insecure) ret = _download_file(server, port, insecure)
print("_download ret={}".format(ret)) log_debug("_download ret={}".format(ret))
if ret == 0: if ret == 0:
_do_reset(True) _do_reset(True)
_run_command("modprobe br_netfilter") _run_command("modprobe br_netfilter")
# Copy flannel.conf
_run_command("mkdir -p {}".format(CNI_DIR))
_run_command("cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR))
(ret, _, _) = _run_command("systemctl start kubelet") (ret, _, _) = _run_command("systemctl start kubelet")
if ret == 0: if ret == 0:
(ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format( (ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
KUBE_ADMIN_CONF, device_info.get_hostname()), timeout=60) KUBE_ADMIN_CONF, device_info.get_hostname()), timeout=60)
print("ret = {}".format(ret)) log_debug("ret = {}".format(ret))
except IOError as e: except IOError as e:
err = "Download failed: {}".format(str(e)) err = "Download failed: {}".format(str(e))

28
src/sonic-ctrmgrd/tests/kube_commands_test.py
View File

@ -13,6 +13,8 @@ import kube_commands
KUBE_ADMIN_CONF = "/tmp/kube_admin.conf" KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
FLANNEL_CONF_FILE = "/tmp/flannel.conf"
CNI_DIR = "/tmp/cni/net.d"
# kube_commands test cases # kube_commands test cases
# NOTE: Ensure state-db entry is complete in PRE as we need to # NOTE: Ensure state-db entry is complete in PRE as we need to
@ -108,9 +110,11 @@ join_test_data = {
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"systemctl stop kubelet", "systemctl stop kubelet",
"modprobe br_netfilter", "modprobe br_netfilter",
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet", "systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name None".format( "kubeadm join --discovery-file {} --node-name None".format(
KUBE_ADMIN_CONF) KUBE_ADMIN_CONF)
@ -129,9 +133,11 @@ None".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"systemctl stop kubelet", "systemctl stop kubelet",
"modprobe br_netfilter", "modprobe br_netfilter",
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet", "systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name None".format( "kubeadm join --discovery-file {} --node-name None".format(
KUBE_ADMIN_CONF) KUBE_ADMIN_CONF)
@ -159,9 +165,11 @@ None".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"systemctl stop kubelet", "systemctl stop kubelet",
"modprobe br_netfilter", "modprobe br_netfilter",
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet", "systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name None".format( "kubeadm join --discovery-file {} --node-name None".format(
KUBE_ADMIN_CONF) KUBE_ADMIN_CONF)
@ -181,9 +189,11 @@ None".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"systemctl stop kubelet", "systemctl stop kubelet",
"modprobe br_netfilter", "modprobe br_netfilter",
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet", "systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name None".format( "kubeadm join --discovery-file {} --node-name None".format(
KUBE_ADMIN_CONF) KUBE_ADMIN_CONF)
@ -213,7 +223,7 @@ reset_test_data = {
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"rm -f {}".format(KUBE_ADMIN_CONF), "rm -f {}".format(KUBE_ADMIN_CONF),
"systemctl stop kubelet" "systemctl stop kubelet"
] ]
@ -228,7 +238,7 @@ None".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} --request-timeout 20s delete node \ "kubectl --kubeconfig {} --request-timeout 20s delete node \
None".format(KUBE_ADMIN_CONF), None".format(KUBE_ADMIN_CONF),
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"rm -f {}".format(KUBE_ADMIN_CONF), "rm -f {}".format(KUBE_ADMIN_CONF),
"systemctl stop kubelet" "systemctl stop kubelet"
] ]
@ -239,7 +249,7 @@ None".format(KUBE_ADMIN_CONF),
common_test.ARGS: [True], common_test.ARGS: [True],
common_test.PROC_CMD: [ common_test.PROC_CMD: [
"kubeadm reset -f", "kubeadm reset -f",
"rm -rf /etc/cni/net.d", "rm -rf {}".format(CNI_DIR),
"rm -f {}".format(KUBE_ADMIN_CONF), "rm -f {}".format(KUBE_ADMIN_CONF),
"systemctl stop kubelet" "systemctl stop kubelet"
] ]
@ -269,7 +279,11 @@ clusters:\n\
kubelet_yaml = "/tmp/kubelet_config.yaml" kubelet_yaml = "/tmp/kubelet_config.yaml"
with open(kubelet_yaml, "w") as s: with open(kubelet_yaml, "w") as s:
s.close() s.close()
with open(FLANNEL_CONF_FILE, "w") as s:
s.close()
kube_commands.KUBELET_YAML = kubelet_yaml kube_commands.KUBELET_YAML = kubelet_yaml
kube_commands.CNI_DIR = CNI_DIR
kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file) kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF