Make client indentity by AME cert (#11946)

* Make client indentity by AME cert

* Join k8s cluster by ipv6

* Change join test cases

* Test case bug fix

* Improve read node label func

* Configure kubelet and change test cases

* For kubernetes version 1.22.2

* Fix undefine issue

Signed-off-by: Yun Li <yunli1@microsoft.com>

files/build_templates/sonic_debian_extension.j2

@@ -455,7 +455,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd
 {% if include_kubernetes == "y" %}
 # Point to kubelet to /etc/resolv.conf
 #
-echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a  $FILESYSTEM_ROOT/etc/default/kubelet
+echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf --cgroup-driver=cgroupfs --node-ip=::"' | sudo tee -a  $FILESYSTEM_ROOT/etc/default/kubelet
 
 # Copy Flannel conf file into sonic-templates
 #

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

@@ -104,10 +104,12 @@ def log_debug(m):
 
 
 def log_error(m):
+    msg = "{}: {}".format(inspect.stack()[1][3], m)
     syslog.syslog(syslog.LOG_ERR, msg)
 
 
 def log_info(m):
+    msg = "{}: {}".format(inspect.stack()[1][3], m)
     syslog.syslog(syslog.LOG_INFO, msg)
 
 

src/sonic-ctrmgrd/ctrmgr/kube_commands.py

@@ -13,10 +13,14 @@ import sys
 import syslog
 import tempfile
 import urllib.request
+import base64
 from urllib.parse import urlparse
 
 import yaml
+import requests
 from sonic_py_common import device_info
+from jinja2 import Template
+from swsscommon import swsscommon
 
 KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
 KUBELET_YAML = "/var/lib/kubelet/config.yaml"
@@ -24,6 +28,9 @@ SERVER_ADMIN_URL = "https://{}/admin.conf"
 LOCK_FILE = "/var/lock/kube_join.lock"
 FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
 CNI_DIR = "/etc/cni/net.d"
+K8S_CA_URL = "https://{}:{}/api/v1/namespaces/default/configmaps/kube-root-ca.crt"
+AME_CRT = "/etc/sonic/credentials/restapiserver.crt"
+AME_KEY = "/etc/sonic/credentials/restapiserver.key"
 
 def log_debug(m):
     msg = "{}: {}".format(inspect.stack()[1][3], m)
@@ -77,8 +84,7 @@ def _run_command(cmd, timeout=5):
 
 def kube_read_labels():
     """ Read current labels on node and return as dict. """
-    KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes --show-labels |\
+    KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes {} --show-labels |tr -s ' ' | cut -f6 -d' '"
- grep {} | tr -s ' ' | cut -f6 -d' '"
 
     labels = {}
     ret, out, _ = _run_command(KUBECTL_GET_CMD.format(
@@ -211,6 +217,68 @@ def _download_file(server, port, insecure):
     log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
 
 
+def _gen_cli_kubeconf(server, port, insecure):
+    """generate identity which can help authenticate and 
+       authorization to k8s cluster
+    """
+    client_kubeconfig_template = """
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: {{ k8s_ca }}
+    server: https://{{ vip }}:{{ port }}
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: user
+  name: user@kubernetes
+current-context: user@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: user
+  user:
+    client-certificate-data: {{ ame_crt }}
+    client-key-data: {{ ame_key }}
+    """
+    if insecure:
+        r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY), verify=False)
+    else:
+        r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY))
+    if not r.ok:
+        raise requests.RequestException("Something wrong with AME cert or something wrong about sonic role in k8s cluster")
+    k8s_ca = r.json()["data"]["ca.crt"]
+    k8s_ca_b64 = base64.b64encode(k8s_ca.encode("utf-8")).decode("utf-8")
+    ame_crt_raw = open(AME_CRT, "rb")
+    ame_crt_b64 = base64.b64encode(ame_crt_raw.read()).decode("utf-8")
+    ame_key_raw = open(AME_KEY, "rb")
+    ame_key_b64 = base64.b64encode(ame_key_raw.read()).decode("utf-8")
+    client_kubeconfig_template_j2 = Template(client_kubeconfig_template)
+    client_kubeconfig = client_kubeconfig_template_j2.render(
+        k8s_ca=k8s_ca_b64, vip=server, port=port, ame_crt=ame_crt_b64, ame_key=ame_key_b64)
+    (h, fname) = tempfile.mkstemp(suffix="_kube_join")
+    os.write(h, client_kubeconfig.encode("utf-8"))
+    os.close(h)
+    log_debug("Downloaded = {}".format(fname))
+
+    shutil.copyfile(fname, KUBE_ADMIN_CONF)
+
+    log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
+
+
+def _get_local_ipv6():
+    try:
+        config_db = swsscommon.DBConnector("CONFIG_DB", 0)
+        mgmt_ip_data = swsscommon.Table(config_db, 'MGMT_INTERFACE')
+        for key in mgmt_ip_data.getKeys():
+            if key.find(":") >= 0:
+                return key.split("|")[1].split("/")[0]
+        raise IOError("IPV6 not find from MGMT_INTERFACE table")
+    except Exception as e:
+        raise IOError(str(e))
+
+
 def _troubleshoot_tips():
     """ log troubleshoot tips which could be handy,
         when in trouble with join
@@ -264,12 +332,14 @@ def _do_reset(pending_join = False):
 
 
 def _do_join(server, port, insecure):
-    KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {}"
+    KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {} --apiserver-advertise-address {}"
     err = ""
     out = ""
     ret = 0
     try:
-        _download_file(server, port, insecure)
+        local_ipv6 = _get_local_ipv6()
+        #_download_file(server, port, insecure)
+        _gen_cli_kubeconf(server, port, insecure)
         _do_reset(True)
         _run_command("modprobe br_netfilter")
         # Copy flannel.conf
@@ -279,11 +349,11 @@ def _do_join(server, port, insecure):
 
         if ret == 0:
             (ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
-                KUBE_ADMIN_CONF, get_device_name()), timeout=60)
+                KUBE_ADMIN_CONF, get_device_name(), local_ipv6), timeout=60)
             log_debug("ret = {}".format(ret))
 
     except IOError as e:
-        err = "Download failed: {}".format(str(e))
+        err = "Join failed: {}".format(str(e))
         ret = -1
         out = ""
 

src/sonic-ctrmgrd/tests/common_test.py

@@ -9,6 +9,7 @@ import time
 CONFIG_DB_NO = 4
 STATE_DB_NO = 6
 FEATURE_TABLE = "FEATURE"
+MGMT_INTERFACE_TABLE = "MGMT_INTERFACE"
 KUBE_LABEL_TABLE = "KUBE_LABELS"
 KUBE_LABEL_SET_KEY = "SET"
 
@@ -41,6 +42,7 @@ KUBE_RETURN = "kube_return"
 IMAGE_TAG = "image_tag"
 FAIL_LOCK = "fail_lock"
 DO_JOIN = "do_join"
+REQ = "req"
 
 # subproc key words
 
@@ -643,8 +645,27 @@ def mock_subproc_side_effect(cmd, shell=False, stdout=None, stderr=None):
     return mock_proc(cmd, index)
 
 
-def set_kube_mock(mock_subproc):
+class mock_reqget:
+    def __init__(self):
+        self.ok = True
+
+    def json(self):
+        return current_test_data.get(REQ, "")
+
+
+def mock_reqget_side_effect(url, cert, verify=True):
+    return mock_reqget()
+
+
+def set_kube_mock(mock_subproc, mock_table=None, mock_conn=None, mock_reqget=None):
     mock_subproc.side_effect = mock_subproc_side_effect
+    if mock_table != None:
+        mock_table.side_effect = table_side_effect
+    if mock_conn != None:
+        mock_conn.side_effect = conn_side_effect
+    if mock_reqget != None:
+        mock_reqget.side_effect = mock_reqget_side_effect
+
 
 def create_remote_ctr_config_json():
     str_conf = '\

src/sonic-ctrmgrd/tests/kube_commands_test.py

@@ -15,6 +15,8 @@ import kube_commands
 KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
 FLANNEL_CONF_FILE = "/tmp/flannel.conf"
 CNI_DIR = "/tmp/cni/net.d"
+AME_CRT = "/tmp/restapiserver.crt"
+AME_KEY = "/tmp/restapiserver.key"
 
 # kube_commands test cases
 # NOTE: Ensure state-db entry is complete in PRE as we need to
@@ -25,8 +27,7 @@ read_labels_test_data = {
         common_test.DESCR: "read labels",
         common_test.RETVAL: 0,
         common_test.PROC_CMD: ["\
-kubectl --kubeconfig {} get nodes --show-labels |\
+kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
         common_test.PROC_OUT: ["foo=bar,hello=world"],
         common_test.POST: {
             "foo": "bar",
@@ -39,8 +40,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\
         common_test.TRIGGER_THROW: True,
         common_test.RETVAL: -1,
         common_test.PROC_CMD: ["\
-kubectl --kubeconfig {} get nodes --show-labels |\
+kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
         common_test.POST: {
         },
         common_test.PROC_KILLED: 1
@@ -49,8 +49,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\
         common_test.DESCR: "read labels fail",
         common_test.RETVAL: -1,
         common_test.PROC_CMD: ["\
-kubectl --kubeconfig {} get nodes --show-labels |\
+kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
         common_test.PROC_OUT: [""],
         common_test.PROC_ERR: ["command failed"],
         common_test.POST: {
@@ -65,8 +64,7 @@ write_labels_test_data = {
         common_test.RETVAL: 0,
         common_test.ARGS: { "foo": "bar", "hello": "World!", "test": "ok" },
         common_test.PROC_CMD: [
-"kubectl --kubeconfig {} get nodes --show-labels |\
+"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
 "kubectl --kubeconfig {} label --overwrite nodes none hello-".format(
     KUBE_ADMIN_CONF),
 "kubectl --kubeconfig {} label --overwrite nodes none hello=World! test=ok".format(
@@ -79,8 +77,7 @@ write_labels_test_data = {
         common_test.RETVAL: 0,
         common_test.ARGS: { "foo": "bar", "hello": "world" },
         common_test.PROC_CMD: [
-"kubectl --kubeconfig {} get nodes --show-labels |\
+"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
  ],
         common_test.PROC_OUT: ["foo=bar,hello=world"]
     },
@@ -90,8 +87,7 @@ write_labels_test_data = {
         common_test.ARGS: { "any": "thing" },
         common_test.RETVAL: -1,
         common_test.PROC_CMD: [
-"kubectl --kubeconfig {} get nodes --show-labels |\
+"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
- grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
 ],
         common_test.PROC_ERR: ["read failed"]
     }
@@ -114,10 +110,22 @@ none".format(KUBE_ADMIN_CONF),
             "mkdir -p {}".format(CNI_DIR),
             "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
-            "kubeadm join --discovery-file {} --node-name none".format(
+            "kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
                 KUBE_ADMIN_CONF)
         ],
-        common_test.PROC_RUN: [True, True]
+        common_test.PROC_RUN: [True, True],
+        common_test.PRE: {
+            common_test.CONFIG_DB_NO: {
+                common_test.MGMT_INTERFACE_TABLE: {
+                    "eth0|FC00:2::32/64": {
+                        "gwaddr": "fc00:2::1"
+                    }
+                }
+            }
+        },
+        common_test.REQ: {
+            "data": {"ca.crt": "test"}
+        }
     },
     1: {
         common_test.DESCR: "Regular secure join",
@@ -135,10 +143,22 @@ none".format(KUBE_ADMIN_CONF),
             "mkdir -p {}".format(CNI_DIR),
             "cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
             "systemctl start kubelet",
-            "kubeadm join --discovery-file {} --node-name none".format(
+            "kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
                 KUBE_ADMIN_CONF)
         ],
-        common_test.PROC_RUN: [True, True]
+        common_test.PROC_RUN: [True, True],
+        common_test.PRE: {
+            common_test.CONFIG_DB_NO: {
+                common_test.MGMT_INTERFACE_TABLE: {
+                    "eth0|FC00:2::32/64": {
+                        "gwaddr": "fc00:2::1"
+                    }
+                }
+            }
+        },
+        common_test.REQ: {
+            "data": {"ca.crt": "test"}
+        }
     },
     2: {
         common_test.DESCR: "Skip join as already connected",
@@ -228,11 +248,17 @@ clusters:\n\
             s.close()
         with open(FLANNEL_CONF_FILE, "w") as s:
             s.close()
+        with open(AME_CRT, "w") as s:
+            s.close()
+        with open(AME_KEY, "w") as s:
+            s.close()
         kube_commands.KUBELET_YAML = kubelet_yaml
         kube_commands.CNI_DIR = CNI_DIR
         kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
         kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
         kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF
+        kube_commands.AME_CRT = AME_CRT
+        kube_commands.AME_KEY = AME_KEY
 
 
     @patch("kube_commands.subprocess.Popen")
@@ -295,11 +321,13 @@ clusters:\n\
                     json.dumps(labels, indent=4)))
                 assert False
 
-
+    @patch("kube_commands.requests.get")
+    @patch("kube_commands.swsscommon.DBConnector")
+    @patch("kube_commands.swsscommon.Table")
     @patch("kube_commands.subprocess.Popen")
-    def test_join(self, mock_subproc):
+    def test_join(self, mock_subproc, mock_table, mock_conn, mock_reqget):
         self.init()
-        common_test.set_kube_mock(mock_subproc)
+        common_test.set_kube_mock(mock_subproc, mock_table, mock_conn, mock_reqget)
 
         for (i, ct_data) in join_test_data.items():
             lock_file = ""