Add k8s master feature (#11637)
* Add k8s master feature Signed-off-by: Yun Li <yunli1@microsoft.com> * Update kubernetes version mistake and make variable passing clear Signed-off-by: Yun Li <yunli1@microsoft.com> * Add CRI-dockerd package Signed-off-by: Yun Li <yunli1@microsoft.com> * Update version variable passing logic Signed-off-by: Yun Li <yunli1@microsoft.com> * Upgrade the worker kubernetes version Signed-off-by: Yun Li <yunli1@microsoft.com> * Install xml file parse tool Signed-off-by: Yun Li <yunli1@microsoft.com> Signed-off-by: Yun Li <yunli1@microsoft.com>
This commit is contained in:
lixiaoyuner
GitHub
parent
743625c2b4
commit
8d6431e754
@ -10,6 +10,7 @@
|
||||
# * ENABLE_ZTP: Enables zero touch provisioning.
|
||||
# * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
|
||||
# * INCLUDE_KUBERNETES: Allows including Kubernetes
|
||||
# * INCLUDE_KUBERNETES_MASTER: Allows including Kubernetes master
|
||||
# * INCLUDE_MUX: Include MUX feature/services for TOR switch.
|
||||
# * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
|
||||
# * by default for TOR switch.
|
||||
@ -366,6 +367,7 @@ SONIC_BUILD_INSTRUCTION := make \
|
||||
KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
|
||||
KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \
|
||||
K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \
|
||||
INCLUDE_KUBERNETES_MASTER=$(INCLUDE_KUBERNETES_MASTER) \
|
||||
SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \
|
||||
SONIC_ENABLE_SYNCD_RPC=$(ENABLE_SYNCD_RPC) \
|
||||
SONIC_INSTALL_DEBUG_TOOLS=$(INSTALL_DEBUG_TOOLS) \
|
||||
|
||||
@ -253,24 +253,55 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install docker-ce=${DOCKER_VERSIO
|
||||
# pip version of 'PyGObject' will be installed during installation of 'sonic-host-services'
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2 python3-gi
|
||||
|
||||
if [ "$INCLUDE_KUBERNETES" == "y" ]
|
||||
then
|
||||
## Install Kubernetes
|
||||
echo '[INFO] Install kubernetes'
|
||||
install_kubernetes () {
|
||||
local ver="$1"
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
|
||||
https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
|
||||
## Check out the sources list update matches current Debian version
|
||||
sudo cp files/image_config/kubernetes/kubernetes.list $FILESYSTEM_ROOT/etc/apt/sources.list.d/
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubernetes-cni=${KUBERNETES_CNI_VERSION}-00
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${KUBERNETES_VERSION}-00
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${KUBERNETES_VERSION}-00
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${KUBERNETES_VERSION}-00
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${ver}
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${ver}
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${ver}
|
||||
}
|
||||
|
||||
if [ "$INCLUDE_KUBERNETES" == "y" ]
|
||||
then
|
||||
## Install Kubernetes
|
||||
echo '[INFO] Install kubernetes'
|
||||
install_kubernetes ${KUBERNETES_VERSION}
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubernetes-cni=${KUBERNETES_CNI_VERSION}
|
||||
else
|
||||
echo '[INFO] Skipping Install kubernetes'
|
||||
fi
|
||||
|
||||
if [ "$INCLUDE_KUBERNETES_MASTER" == "y" ]
|
||||
then
|
||||
## Install Kubernetes master
|
||||
echo '[INFO] Install kubernetes master'
|
||||
install_kubernetes ${MASTER_KUBERNETES_VERSION}
|
||||
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
|
||||
https://packages.microsoft.com/keys/microsoft.asc | \
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
|
||||
https://packages.microsoft.com/keys/msopentech.asc | \
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
|
||||
echo "deb [arch=amd64] https://packages.microsoft.com/repos/azurecore-debian $IMAGE_DISTRO main" | \
|
||||
sudo tee $FILESYSTEM_ROOT/etc/apt/sources.list.d/azure.list
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install metricsext2
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \
|
||||
https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/cri-dockerd.deb
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT rm -f /tmp/cri-dockerd.deb
|
||||
else
|
||||
echo '[INFO] Skipping Install kubernetes master'
|
||||
fi
|
||||
|
||||
## Add docker config drop-in to specify dockerd command line
|
||||
sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/
|
||||
## Note: $_ means last argument of last command
|
||||
|
||||
@ -731,6 +731,27 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIV
|
||||
echo "docker images pull complete"
|
||||
{% endif %}
|
||||
|
||||
{% if include_kubernetes_master == "y" %}
|
||||
# Pull in kubernetes master docker images
|
||||
echo "pulling kubernetes master images ..."
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/pause:${MASTER_PAUSE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-apiserver:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-controller-manager:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-scheduler:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-proxy:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/coredns/coredns:${MASTER_COREDNS_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/etcd:${MASTER_ETCD_VERSION}
|
||||
echo "kubernetes master docker images pull complete"
|
||||
# Install python package for mdm service usage
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install statsd
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable mdm.service
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubelet.service
|
||||
# Add kubernetes master entrance
|
||||
sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/sbin/
|
||||
sudo sed -i '/^exit 0/i\bash /usr/sbin/kubernetes_master_entrance.sh' $FILESYSTEM_ROOT/etc/rc.local
|
||||
{% endif %}
|
||||
|
||||
{% macro get_install_options(set_owner, enabled) -%}
|
||||
{% set args = ["-y", "-v", "DEBUG"] -%}
|
||||
{% if set_owner -%}
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
# This script is for kubernetes master image usage
|
||||
# Will mount kubernetes master disk and execute kubernetes entrance script
|
||||
21
rules/config
21
rules/config
@ -174,9 +174,24 @@ KUBE_DOCKER_PROXY = http://172.16.1.1:3128/
|
||||
# These are Used *only* when INCLUDE_KUBERNETES=y
|
||||
# NOTE: As a worker node it has to run version compatible to kubernetes master.
|
||||
#
|
||||
KUBERNETES_VERSION = 1.21.1
|
||||
KUBERNETES_CNI_VERSION = 0.8.7
|
||||
K8s_GCR_IO_PAUSE_VERSION = 3.4.1
|
||||
KUBERNETES_VERSION = 1.22.2-00
|
||||
KUBERNETES_CNI_VERSION = 0.8.7-00
|
||||
K8s_GCR_IO_PAUSE_VERSION = 3.5
|
||||
|
||||
# INCLUDE_KUBERNETES_MASTER - if set to y kubernetes packages are installed o be able
|
||||
# to run as master node in kubernetes cluster
|
||||
INCLUDE_KUBERNETES_MASTER ?= n
|
||||
|
||||
# MASTER_KUBERNETES_VERSION - version of k8s components
|
||||
# MASTER_PAUSE_VERSION - version of pause container image
|
||||
# MASTER_COREDNS_VERSION - version of coredns container image
|
||||
# MASTER_ETCD_VERSION = version of etcd container image
|
||||
MASTER_KUBERNETES_VERSION = 1.22.2-00
|
||||
MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION = v1.22.2
|
||||
MASTER_PAUSE_VERSION = 3.5
|
||||
MASTER_COREDNS_VERSION = v1.8.4
|
||||
MASTER_ETCD_VERSION = 3.5.0-0
|
||||
MASTER_CRI_DOCKERD = 0.2.5
|
||||
|
||||
# SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
|
||||
# To not use the auto-generated self-signed certificate, the required files to sign the image as below:
|
||||
|
||||
8
slave.mk
8
slave.mk
@ -385,6 +385,7 @@ $(info "INCLUDE_NAT" : "$(INCLUDE_NAT)")
|
||||
$(info "INCLUDE_DHCP_RELAY" : "$(INCLUDE_DHCP_RELAY)")
|
||||
$(info "INCLUDE_P4RT" : "$(INCLUDE_P4RT)")
|
||||
$(info "INCLUDE_KUBERNETES" : "$(INCLUDE_KUBERNETES)")
|
||||
$(info "INCLUDE_KUBERNETES_MASTER" : "$(INCLUDE_KUBERNETES_MASTER)")
|
||||
$(info "INCLUDE_MACSEC" : "$(INCLUDE_MACSEC)")
|
||||
$(info "INCLUDE_MUX" : "$(INCLUDE_MUX)")
|
||||
$(info "INCLUDE_BOOTCHART : "$(INCLUDE_BOOTCHART)")
|
||||
@ -1188,6 +1189,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
|
||||
export shutdown_bgp_on_start="$(SHUTDOWN_BGP_ON_START)"
|
||||
export default_buffer_model="$(SONIC_BUFFER_MODEL)"
|
||||
export include_kubernetes="$(INCLUDE_KUBERNETES)"
|
||||
export include_kubernetes_master="$(INCLUDE_KUBERNETES_MASTER)"
|
||||
export kube_docker_proxy="$(KUBE_DOCKER_PROXY)"
|
||||
export enable_pfcwd_on_start="$(ENABLE_PFCWD_ON_START)"
|
||||
export installer_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($*_INSTALLS) $(FIPS_BASEIMAGE_INSTALLERS))"
|
||||
@ -1358,6 +1360,12 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
|
||||
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
|
||||
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
|
||||
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
|
||||
MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \
|
||||
MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION=$(MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION) \
|
||||
MASTER_PAUSE_VERSION=$(MASTER_PAUSE_VERSION) \
|
||||
MASTER_COREDNS_VERSION=$(MASTER_COREDNS_VERSION) \
|
||||
MASTER_ETCD_VERSION=$(MASTER_ETCD_VERSION) \
|
||||
MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
|
||||
./build_debian.sh $(LOG)
|
||||
|
||||
USERNAME="$(USERNAME)" \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user