matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Make client indentity by AME cert (#11946) (#12908)

Browse Source
This commit is contained in:
mssonicbld 2022-12-02 13:13:26 +08:00 committed by GitHub
parent 02345963db
commit 7152e84277
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 148 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
files/build_templates/sonic_debian_extension.j2
View File

@ -446,7 +446,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd
{% if include_kubernetes == "y" %}
# Point to kubelet to /etc/resolv.conf
#
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf --cgroup-driver=cgroupfs --node-ip=::"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
# Copy Flannel conf file into sonic-templates
#

2
src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
View File

@ -104,10 +104,12 @@ def log_debug(m):
def log_error(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
syslog.syslog(syslog.LOG_ERR, msg)
def log_info(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
syslog.syslog(syslog.LOG_INFO, msg)

82
src/sonic-ctrmgrd/ctrmgr/kube_commands.py
View File

@ -13,10 +13,14 @@ import sys
import syslog
import tempfile
import urllib.request
import base64
from urllib.parse import urlparse
import yaml
import requests
from sonic_py_common import device_info
from jinja2 import Template
from swsscommon import swsscommon
KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
KUBELET_YAML = "/var/lib/kubelet/config.yaml"
@ -24,6 +28,9 @@ SERVER_ADMIN_URL = "https://{}/admin.conf"
LOCK_FILE = "/var/lock/kube_join.lock"
FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
CNI_DIR = "/etc/cni/net.d"
K8S_CA_URL = "https://{}:{}/api/v1/namespaces/default/configmaps/kube-root-ca.crt"
AME_CRT = "/etc/sonic/credentials/restapiserver.crt"
AME_KEY = "/etc/sonic/credentials/restapiserver.key"
def log_debug(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
@ -77,8 +84,7 @@ def _run_command(cmd, timeout=5):
def kube_read_labels():
""" Read current labels on node and return as dict. """
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes --show-labels |\
grep {} | tr -s ' ' | cut -f6 -d' '"
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes {} --show-labels |tr -s ' ' | cut -f6 -d' '"
labels = {}
ret, out, _ = _run_command(KUBECTL_GET_CMD.format(
@ -211,6 +217,68 @@ def _download_file(server, port, insecure):
log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
def _gen_cli_kubeconf(server, port, insecure):
"""generate identity which can help authenticate and
authorization to k8s cluster
"""
client_kubeconfig_template = """
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: {{ k8s_ca }}
server: https://{{ vip }}:{{ port }}
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: user
name: user@kubernetes
current-context: user@kubernetes
kind: Config
preferences: {}
users:
- name: user
user:
client-certificate-data: {{ ame_crt }}
client-key-data: {{ ame_key }}
"""
if insecure:
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY), verify=False)
else:
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY))
if not r.ok:
raise requests.RequestException("Something wrong with AME cert or something wrong about sonic role in k8s cluster")
k8s_ca = r.json()["data"]["ca.crt"]
k8s_ca_b64 = base64.b64encode(k8s_ca.encode("utf-8")).decode("utf-8")
ame_crt_raw = open(AME_CRT, "rb")
ame_crt_b64 = base64.b64encode(ame_crt_raw.read()).decode("utf-8")
ame_key_raw = open(AME_KEY, "rb")
ame_key_b64 = base64.b64encode(ame_key_raw.read()).decode("utf-8")
client_kubeconfig_template_j2 = Template(client_kubeconfig_template)
client_kubeconfig = client_kubeconfig_template_j2.render(
k8s_ca=k8s_ca_b64, vip=server, port=port, ame_crt=ame_crt_b64, ame_key=ame_key_b64)
(h, fname) = tempfile.mkstemp(suffix="_kube_join")
os.write(h, client_kubeconfig.encode("utf-8"))
os.close(h)
log_debug("Downloaded = {}".format(fname))
shutil.copyfile(fname, KUBE_ADMIN_CONF)
log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
def _get_local_ipv6():
try:
config_db = swsscommon.DBConnector("CONFIG_DB", 0)
mgmt_ip_data = swsscommon.Table(config_db, 'MGMT_INTERFACE')
for key in mgmt_ip_data.getKeys():
if key.find(":") >= 0:
return key.split("|")[1].split("/")[0]
raise IOError("IPV6 not find from MGMT_INTERFACE table")
except Exception as e:
raise IOError(str(e))
def _troubleshoot_tips():
""" log troubleshoot tips which could be handy,
when in trouble with join
@ -264,12 +332,14 @@ def _do_reset(pending_join = False):
def _do_join(server, port, insecure):
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {}"
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {} --apiserver-advertise-address {}"
err = ""
out = ""
ret = 0
try:
_download_file(server, port, insecure)
local_ipv6 = _get_local_ipv6()
#_download_file(server, port, insecure)
_gen_cli_kubeconf(server, port, insecure)
_do_reset(True)
_run_command("modprobe br_netfilter")
# Copy flannel.conf
@ -279,11 +349,11 @@ def _do_join(server, port, insecure):
if ret == 0:
(ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
KUBE_ADMIN_CONF, get_device_name()), timeout=60)
KUBE_ADMIN_CONF, get_device_name(), local_ipv6), timeout=60)
log_debug("ret = {}".format(ret))
except IOError as e:
err = "Download failed: {}".format(str(e))
err = "Join failed: {}".format(str(e))
ret = -1
out = ""

23
src/sonic-ctrmgrd/tests/common_test.py
View File

@ -9,6 +9,7 @@ import time
CONFIG_DB_NO = 4
STATE_DB_NO = 6
FEATURE_TABLE = "FEATURE"
MGMT_INTERFACE_TABLE = "MGMT_INTERFACE"
KUBE_LABEL_TABLE = "KUBE_LABELS"
KUBE_LABEL_SET_KEY = "SET"
@ -41,6 +42,7 @@ KUBE_RETURN = "kube_return"
IMAGE_TAG = "image_tag"
FAIL_LOCK = "fail_lock"
DO_JOIN = "do_join"
REQ = "req"
# subproc key words
@ -643,8 +645,27 @@ def mock_subproc_side_effect(cmd, shell=False, stdout=None, stderr=None):
return mock_proc(cmd, index)
def set_kube_mock(mock_subproc):
class mock_reqget:
def __init__(self):
self.ok = True
def json(self):
return current_test_data.get(REQ, "")
def mock_reqget_side_effect(url, cert, verify=True):
return mock_reqget()
def set_kube_mock(mock_subproc, mock_table=None, mock_conn=None, mock_reqget=None):
mock_subproc.side_effect = mock_subproc_side_effect
if mock_table != None:
mock_table.side_effect = table_side_effect
if mock_conn != None:
mock_conn.side_effect = conn_side_effect
if mock_reqget != None:
mock_reqget.side_effect = mock_reqget_side_effect
def create_remote_ctr_config_json():
str_conf = '\

66
src/sonic-ctrmgrd/tests/kube_commands_test.py
View File

@ -15,6 +15,8 @@ import kube_commands
KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
FLANNEL_CONF_FILE = "/tmp/flannel.conf"
CNI_DIR = "/tmp/cni/net.d"
AME_CRT = "/tmp/restapiserver.crt"
AME_KEY = "/tmp/restapiserver.key"
# kube_commands test cases
# NOTE: Ensure state-db entry is complete in PRE as we need to
@ -25,8 +27,7 @@ read_labels_test_data = {
common_test.DESCR: "read labels",
common_test.RETVAL: 0,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.PROC_OUT: ["foo=bar,hello=world"],
common_test.POST: {
"foo": "bar",
@ -39,8 +40,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\
common_test.TRIGGER_THROW: True,
common_test.RETVAL: -1,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.POST: {
},
common_test.PROC_KILLED: 1
@ -49,8 +49,7 @@ kubectl --kubeconfig {} get nodes --show-labels |\
common_test.DESCR: "read labels fail",
common_test.RETVAL: -1,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.PROC_OUT: [""],
common_test.PROC_ERR: ["command failed"],
common_test.POST: {
@ -65,8 +64,7 @@ write_labels_test_data = {
common_test.RETVAL: 0,
common_test.ARGS: { "foo": "bar", "hello": "World!", "test": "ok" },
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} label --overwrite nodes none hello-".format(
KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} label --overwrite nodes none hello=World! test=ok".format(
@ -79,8 +77,7 @@ write_labels_test_data = {
common_test.RETVAL: 0,
common_test.ARGS: { "foo": "bar", "hello": "world" },
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
],
common_test.PROC_OUT: ["foo=bar,hello=world"]
},
@ -90,8 +87,7 @@ write_labels_test_data = {
common_test.ARGS: { "any": "thing" },
common_test.RETVAL: -1,
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
],
common_test.PROC_ERR: ["read failed"]
}
@ -114,10 +110,22 @@ none".format(KUBE_ADMIN_CONF),
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name none".format(
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
KUBE_ADMIN_CONF)
],
common_test.PROC_RUN: [True, True]
common_test.PROC_RUN: [True, True],
common_test.PRE: {
common_test.CONFIG_DB_NO: {
common_test.MGMT_INTERFACE_TABLE: {
"eth0|FC00:2::32/64": {
"gwaddr": "fc00:2::1"
}
}
}
},
common_test.REQ: {
"data": {"ca.crt": "test"}
}
},
1: {
common_test.DESCR: "Regular secure join",
@ -135,10 +143,22 @@ none".format(KUBE_ADMIN_CONF),
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name none".format(
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
KUBE_ADMIN_CONF)
],
common_test.PROC_RUN: [True, True]
common_test.PROC_RUN: [True, True],
common_test.PRE: {
common_test.CONFIG_DB_NO: {
common_test.MGMT_INTERFACE_TABLE: {
"eth0|FC00:2::32/64": {
"gwaddr": "fc00:2::1"
}
}
}
},
common_test.REQ: {
"data": {"ca.crt": "test"}
}
},
2: {
common_test.DESCR: "Skip join as already connected",
@ -228,11 +248,17 @@ clusters:\n\
s.close()
with open(FLANNEL_CONF_FILE, "w") as s:
s.close()
with open(AME_CRT, "w") as s:
s.close()
with open(AME_KEY, "w") as s:
s.close()
kube_commands.KUBELET_YAML = kubelet_yaml
kube_commands.CNI_DIR = CNI_DIR
kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF
kube_commands.AME_CRT = AME_CRT
kube_commands.AME_KEY = AME_KEY
@patch("kube_commands.subprocess.Popen")
@ -295,11 +321,13 @@ clusters:\n\
json.dumps(labels, indent=4)))
assert False
@patch("kube_commands.requests.get")
@patch("kube_commands.swsscommon.DBConnector")
@patch("kube_commands.swsscommon.Table")
@patch("kube_commands.subprocess.Popen")
def test_join(self, mock_subproc):
def test_join(self, mock_subproc, mock_table, mock_conn, mock_reqget):
self.init()
common_test.set_kube_mock(mock_subproc)
common_test.set_kube_mock(mock_subproc, mock_table, mock_conn, mock_reqget)
for (i, ct_data) in join_test_data.items():
lock_file = ""