From 7152e84277e5a3dd9a8a19855b0ecf1c39d62700 Mon Sep 17 00:00:00 2001 From: mssonicbld <79238446+mssonicbld@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:13:26 +0800 Subject: [PATCH] Make client indentity by AME cert (#11946) (#12908) --- .../build_templates/sonic_debian_extension.j2 | 2 +- src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py | 2 + src/sonic-ctrmgrd/ctrmgr/kube_commands.py | 82 +++++++++++++++++-- src/sonic-ctrmgrd/tests/common_test.py | 23 +++++- src/sonic-ctrmgrd/tests/kube_commands_test.py | 66 ++++++++++----- 5 files changed, 148 insertions(+), 27 deletions(-) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 9b56f10872..81af352663 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -446,7 +446,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd {% if include_kubernetes == "y" %} # Point to kubelet to /etc/resolv.conf # -echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet +echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf --cgroup-driver=cgroupfs --node-ip=::"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet # Copy Flannel conf file into sonic-templates # diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py index 3941a05a73..2defb6e453 100755 --- a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py @@ -104,10 +104,12 @@ def log_debug(m): def log_error(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) syslog.syslog(syslog.LOG_ERR, msg) def log_info(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) syslog.syslog(syslog.LOG_INFO, msg) diff --git a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py index 9a6ea9ed8d..3adea36ef1 100755 --- a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py +++ b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py @@ -13,10 +13,14 @@ import sys import syslog import tempfile import urllib.request +import base64 from urllib.parse import urlparse import yaml +import requests from sonic_py_common import device_info +from jinja2 import Template +from swsscommon import swsscommon KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf" KUBELET_YAML = "/var/lib/kubelet/config.yaml" @@ -24,6 +28,9 @@ SERVER_ADMIN_URL = "https://{}/admin.conf" LOCK_FILE = "/var/lock/kube_join.lock" FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist" CNI_DIR = "/etc/cni/net.d" +K8S_CA_URL = "https://{}:{}/api/v1/namespaces/default/configmaps/kube-root-ca.crt" +AME_CRT = "/etc/sonic/credentials/restapiserver.crt" +AME_KEY = "/etc/sonic/credentials/restapiserver.key" def log_debug(m): msg = "{}: {}".format(inspect.stack()[1][3], m) @@ -77,8 +84,7 @@ def _run_command(cmd, timeout=5): def kube_read_labels(): """ Read current labels on node and return as dict. """ - KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes --show-labels |\ - grep {} | tr -s ' ' | cut -f6 -d' '" + KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes {} --show-labels |tr -s ' ' | cut -f6 -d' '" labels = {} ret, out, _ = _run_command(KUBECTL_GET_CMD.format( @@ -211,6 +217,68 @@ def _download_file(server, port, insecure): log_debug("{} downloaded".format(KUBE_ADMIN_CONF)) +def _gen_cli_kubeconf(server, port, insecure): + """generate identity which can help authenticate and + authorization to k8s cluster + """ + client_kubeconfig_template = """ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: {{ k8s_ca }} + server: https://{{ vip }}:{{ port }} + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: user + name: user@kubernetes +current-context: user@kubernetes +kind: Config +preferences: {} +users: +- name: user + user: + client-certificate-data: {{ ame_crt }} + client-key-data: {{ ame_key }} + """ + if insecure: + r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY), verify=False) + else: + r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY)) + if not r.ok: + raise requests.RequestException("Something wrong with AME cert or something wrong about sonic role in k8s cluster") + k8s_ca = r.json()["data"]["ca.crt"] + k8s_ca_b64 = base64.b64encode(k8s_ca.encode("utf-8")).decode("utf-8") + ame_crt_raw = open(AME_CRT, "rb") + ame_crt_b64 = base64.b64encode(ame_crt_raw.read()).decode("utf-8") + ame_key_raw = open(AME_KEY, "rb") + ame_key_b64 = base64.b64encode(ame_key_raw.read()).decode("utf-8") + client_kubeconfig_template_j2 = Template(client_kubeconfig_template) + client_kubeconfig = client_kubeconfig_template_j2.render( + k8s_ca=k8s_ca_b64, vip=server, port=port, ame_crt=ame_crt_b64, ame_key=ame_key_b64) + (h, fname) = tempfile.mkstemp(suffix="_kube_join") + os.write(h, client_kubeconfig.encode("utf-8")) + os.close(h) + log_debug("Downloaded = {}".format(fname)) + + shutil.copyfile(fname, KUBE_ADMIN_CONF) + + log_debug("{} downloaded".format(KUBE_ADMIN_CONF)) + + +def _get_local_ipv6(): + try: + config_db = swsscommon.DBConnector("CONFIG_DB", 0) + mgmt_ip_data = swsscommon.Table(config_db, 'MGMT_INTERFACE') + for key in mgmt_ip_data.getKeys(): + if key.find(":") >= 0: + return key.split("|")[1].split("/")[0] + raise IOError("IPV6 not find from MGMT_INTERFACE table") + except Exception as e: + raise IOError(str(e)) + + def _troubleshoot_tips(): """ log troubleshoot tips which could be handy, when in trouble with join @@ -264,12 +332,14 @@ def _do_reset(pending_join = False): def _do_join(server, port, insecure): - KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {}" + KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {} --apiserver-advertise-address {}" err = "" out = "" ret = 0 try: - _download_file(server, port, insecure) + local_ipv6 = _get_local_ipv6() + #_download_file(server, port, insecure) + _gen_cli_kubeconf(server, port, insecure) _do_reset(True) _run_command("modprobe br_netfilter") # Copy flannel.conf @@ -279,11 +349,11 @@ def _do_join(server, port, insecure): if ret == 0: (ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format( - KUBE_ADMIN_CONF, get_device_name()), timeout=60) + KUBE_ADMIN_CONF, get_device_name(), local_ipv6), timeout=60) log_debug("ret = {}".format(ret)) except IOError as e: - err = "Download failed: {}".format(str(e)) + err = "Join failed: {}".format(str(e)) ret = -1 out = "" diff --git a/src/sonic-ctrmgrd/tests/common_test.py b/src/sonic-ctrmgrd/tests/common_test.py index 6e9763962b..9283e3ad25 100755 --- a/src/sonic-ctrmgrd/tests/common_test.py +++ b/src/sonic-ctrmgrd/tests/common_test.py @@ -9,6 +9,7 @@ import time CONFIG_DB_NO = 4 STATE_DB_NO = 6 FEATURE_TABLE = "FEATURE" +MGMT_INTERFACE_TABLE = "MGMT_INTERFACE" KUBE_LABEL_TABLE = "KUBE_LABELS" KUBE_LABEL_SET_KEY = "SET" @@ -41,6 +42,7 @@ KUBE_RETURN = "kube_return" IMAGE_TAG = "image_tag" FAIL_LOCK = "fail_lock" DO_JOIN = "do_join" +REQ = "req" # subproc key words @@ -643,8 +645,27 @@ def mock_subproc_side_effect(cmd, shell=False, stdout=None, stderr=None): return mock_proc(cmd, index) -def set_kube_mock(mock_subproc): +class mock_reqget: + def __init__(self): + self.ok = True + + def json(self): + return current_test_data.get(REQ, "") + + +def mock_reqget_side_effect(url, cert, verify=True): + return mock_reqget() + + +def set_kube_mock(mock_subproc, mock_table=None, mock_conn=None, mock_reqget=None): mock_subproc.side_effect = mock_subproc_side_effect + if mock_table != None: + mock_table.side_effect = table_side_effect + if mock_conn != None: + mock_conn.side_effect = conn_side_effect + if mock_reqget != None: + mock_reqget.side_effect = mock_reqget_side_effect + def create_remote_ctr_config_json(): str_conf = '\ diff --git a/src/sonic-ctrmgrd/tests/kube_commands_test.py b/src/sonic-ctrmgrd/tests/kube_commands_test.py index 637802ab1d..d8e0939efb 100755 --- a/src/sonic-ctrmgrd/tests/kube_commands_test.py +++ b/src/sonic-ctrmgrd/tests/kube_commands_test.py @@ -15,6 +15,8 @@ import kube_commands KUBE_ADMIN_CONF = "/tmp/kube_admin.conf" FLANNEL_CONF_FILE = "/tmp/flannel.conf" CNI_DIR = "/tmp/cni/net.d" +AME_CRT = "/tmp/restapiserver.crt" +AME_KEY = "/tmp/restapiserver.key" # kube_commands test cases # NOTE: Ensure state-db entry is complete in PRE as we need to @@ -25,8 +27,7 @@ read_labels_test_data = { common_test.DESCR: "read labels", common_test.RETVAL: 0, common_test.PROC_CMD: ["\ -kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], +kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], common_test.PROC_OUT: ["foo=bar,hello=world"], common_test.POST: { "foo": "bar", @@ -39,8 +40,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\ common_test.TRIGGER_THROW: True, common_test.RETVAL: -1, common_test.PROC_CMD: ["\ -kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], +kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], common_test.POST: { }, common_test.PROC_KILLED: 1 @@ -49,8 +49,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\ common_test.DESCR: "read labels fail", common_test.RETVAL: -1, common_test.PROC_CMD: ["\ -kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], +kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)], common_test.PROC_OUT: [""], common_test.PROC_ERR: ["command failed"], common_test.POST: { @@ -65,8 +64,7 @@ write_labels_test_data = { common_test.RETVAL: 0, common_test.ARGS: { "foo": "bar", "hello": "World!", "test": "ok" }, common_test.PROC_CMD: [ -"kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF), +"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF), "kubectl --kubeconfig {} label --overwrite nodes none hello-".format( KUBE_ADMIN_CONF), "kubectl --kubeconfig {} label --overwrite nodes none hello=World! test=ok".format( @@ -79,8 +77,7 @@ write_labels_test_data = { common_test.RETVAL: 0, common_test.ARGS: { "foo": "bar", "hello": "world" }, common_test.PROC_CMD: [ -"kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF) +"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF) ], common_test.PROC_OUT: ["foo=bar,hello=world"] }, @@ -90,8 +87,7 @@ write_labels_test_data = { common_test.ARGS: { "any": "thing" }, common_test.RETVAL: -1, common_test.PROC_CMD: [ -"kubectl --kubeconfig {} get nodes --show-labels |\ - grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF) +"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF) ], common_test.PROC_ERR: ["read failed"] } @@ -114,10 +110,22 @@ none".format(KUBE_ADMIN_CONF), "mkdir -p {}".format(CNI_DIR), "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR), "systemctl start kubelet", - "kubeadm join --discovery-file {} --node-name none".format( + "kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format( KUBE_ADMIN_CONF) ], - common_test.PROC_RUN: [True, True] + common_test.PROC_RUN: [True, True], + common_test.PRE: { + common_test.CONFIG_DB_NO: { + common_test.MGMT_INTERFACE_TABLE: { + "eth0|FC00:2::32/64": { + "gwaddr": "fc00:2::1" + } + } + } + }, + common_test.REQ: { + "data": {"ca.crt": "test"} + } }, 1: { common_test.DESCR: "Regular secure join", @@ -135,10 +143,22 @@ none".format(KUBE_ADMIN_CONF), "mkdir -p {}".format(CNI_DIR), "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR), "systemctl start kubelet", - "kubeadm join --discovery-file {} --node-name none".format( + "kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format( KUBE_ADMIN_CONF) ], - common_test.PROC_RUN: [True, True] + common_test.PROC_RUN: [True, True], + common_test.PRE: { + common_test.CONFIG_DB_NO: { + common_test.MGMT_INTERFACE_TABLE: { + "eth0|FC00:2::32/64": { + "gwaddr": "fc00:2::1" + } + } + } + }, + common_test.REQ: { + "data": {"ca.crt": "test"} + } }, 2: { common_test.DESCR: "Skip join as already connected", @@ -228,11 +248,17 @@ clusters:\n\ s.close() with open(FLANNEL_CONF_FILE, "w") as s: s.close() + with open(AME_CRT, "w") as s: + s.close() + with open(AME_KEY, "w") as s: + s.close() kube_commands.KUBELET_YAML = kubelet_yaml kube_commands.CNI_DIR = CNI_DIR kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file) kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF + kube_commands.AME_CRT = AME_CRT + kube_commands.AME_KEY = AME_KEY @patch("kube_commands.subprocess.Popen") @@ -295,11 +321,13 @@ clusters:\n\ json.dumps(labels, indent=4))) assert False - + @patch("kube_commands.requests.get") + @patch("kube_commands.swsscommon.DBConnector") + @patch("kube_commands.swsscommon.Table") @patch("kube_commands.subprocess.Popen") - def test_join(self, mock_subproc): + def test_join(self, mock_subproc, mock_table, mock_conn, mock_reqget): self.init() - common_test.set_kube_mock(mock_subproc) + common_test.set_kube_mock(mock_subproc, mock_table, mock_conn, mock_reqget) for (i, ct_data) in join_test_data.items(): lock_file = ""