diff --git a/Makefile.work b/Makefile.work index 107134cf09..3f64531ab1 100644 --- a/Makefile.work +++ b/Makefile.work @@ -10,6 +10,7 @@ # * ENABLE_ZTP: Enables zero touch provisioning. # * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart. # * INCLUDE_KUBERNETES: Allows including Kubernetes +# * INCLUDE_KUBERNETES_MASTER: Allows including Kubernetes master # * INCLUDE_MUX: Include MUX feature/services for TOR switch. # * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports # * by default for TOR switch. @@ -366,6 +367,7 @@ SONIC_BUILD_INSTRUCTION := make \ KUBERNETES_VERSION=$(KUBERNETES_VERSION) \ KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \ K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \ + INCLUDE_KUBERNETES_MASTER=$(INCLUDE_KUBERNETES_MASTER) \ SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \ SONIC_ENABLE_SYNCD_RPC=$(ENABLE_SYNCD_RPC) \ SONIC_INSTALL_DEBUG_TOOLS=$(INSTALL_DEBUG_TOOLS) \ diff --git a/build_debian.sh b/build_debian.sh index 7947aef22d..927b051d28 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -253,24 +253,55 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install docker-ce=${DOCKER_VERSIO # pip version of 'PyGObject' will be installed during installation of 'sonic-host-services' sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2 python3-gi -if [ "$INCLUDE_KUBERNETES" == "y" ] -then - ## Install Kubernetes - echo '[INFO] Install kubernetes' +install_kubernetes () { + local ver="$1" sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add - ## Check out the sources list update matches current Debian version sudo cp files/image_config/kubernetes/kubernetes.list $FILESYSTEM_ROOT/etc/apt/sources.list.d/ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubernetes-cni=${KUBERNETES_CNI_VERSION}-00 - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${KUBERNETES_VERSION}-00 - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${KUBERNETES_VERSION}-00 - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${KUBERNETES_VERSION}-00 + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${ver} + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${ver} + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${ver} +} + +if [ "$INCLUDE_KUBERNETES" == "y" ] +then + ## Install Kubernetes + echo '[INFO] Install kubernetes' + install_kubernetes ${KUBERNETES_VERSION} + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubernetes-cni=${KUBERNETES_CNI_VERSION} else echo '[INFO] Skipping Install kubernetes' fi +if [ "$INCLUDE_KUBERNETES_MASTER" == "y" ] +then + ## Install Kubernetes master + echo '[INFO] Install kubernetes master' + install_kubernetes ${MASTER_KUBERNETES_VERSION} + + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ + https://packages.microsoft.com/keys/microsoft.asc | \ + sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add - + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ + https://packages.microsoft.com/keys/msopentech.asc | \ + sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add - + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azurecore-debian $IMAGE_DISTRO main" | \ + sudo tee $FILESYSTEM_ROOT/etc/apt/sources.list.d/azure.list + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install metricsext2 + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \ + https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/cri-dockerd.deb + sudo LANG=C chroot $FILESYSTEM_ROOT rm -f /tmp/cri-dockerd.deb +else + echo '[INFO] Skipping Install kubernetes master' +fi + ## Add docker config drop-in to specify dockerd command line sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/ ## Note: $_ means last argument of last command diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index a15822046b..4b7a77b315 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -731,6 +731,27 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIV echo "docker images pull complete" {% endif %} +{% if include_kubernetes_master == "y" %} +# Pull in kubernetes master docker images +echo "pulling kubernetes master images ..." +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/pause:${MASTER_PAUSE_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-apiserver:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-controller-manager:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-scheduler:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-proxy:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/coredns/coredns:${MASTER_COREDNS_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/etcd:${MASTER_ETCD_VERSION} +echo "kubernetes master docker images pull complete" +# Install python package for mdm service usage +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install statsd +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable mdm.service +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubelet.service +# Add kubernetes master entrance +sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/sbin/ +sudo sed -i '/^exit 0/i\bash /usr/sbin/kubernetes_master_entrance.sh' $FILESYSTEM_ROOT/etc/rc.local +{% endif %} + {% macro get_install_options(set_owner, enabled) -%} {% set args = ["-y", "-v", "DEBUG"] -%} {% if set_owner -%} diff --git a/files/image_config/kubernetes/kubernetes_master_entrance.sh b/files/image_config/kubernetes/kubernetes_master_entrance.sh new file mode 100644 index 0000000000..b7695a61ca --- /dev/null +++ b/files/image_config/kubernetes/kubernetes_master_entrance.sh @@ -0,0 +1,2 @@ +# This script is for kubernetes master image usage +# Will mount kubernetes master disk and execute kubernetes entrance script diff --git a/rules/config b/rules/config index 23622bcc87..380b28bc7c 100644 --- a/rules/config +++ b/rules/config @@ -174,9 +174,24 @@ KUBE_DOCKER_PROXY = http://172.16.1.1:3128/ # These are Used *only* when INCLUDE_KUBERNETES=y # NOTE: As a worker node it has to run version compatible to kubernetes master. # -KUBERNETES_VERSION = 1.21.1 -KUBERNETES_CNI_VERSION = 0.8.7 -K8s_GCR_IO_PAUSE_VERSION = 3.4.1 +KUBERNETES_VERSION = 1.22.2-00 +KUBERNETES_CNI_VERSION = 0.8.7-00 +K8s_GCR_IO_PAUSE_VERSION = 3.5 + +# INCLUDE_KUBERNETES_MASTER - if set to y kubernetes packages are installed o be able +# to run as master node in kubernetes cluster +INCLUDE_KUBERNETES_MASTER ?= n + +# MASTER_KUBERNETES_VERSION - version of k8s components +# MASTER_PAUSE_VERSION - version of pause container image +# MASTER_COREDNS_VERSION - version of coredns container image +# MASTER_ETCD_VERSION = version of etcd container image +MASTER_KUBERNETES_VERSION = 1.22.2-00 +MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION = v1.22.2 +MASTER_PAUSE_VERSION = 3.5 +MASTER_COREDNS_VERSION = v1.8.4 +MASTER_ETCD_VERSION = 3.5.0-0 +MASTER_CRI_DOCKERD = 0.2.5 # SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature # To not use the auto-generated self-signed certificate, the required files to sign the image as below: diff --git a/slave.mk b/slave.mk index 2e648b37f4..e1f4a0ef80 100644 --- a/slave.mk +++ b/slave.mk @@ -385,6 +385,7 @@ $(info "INCLUDE_NAT" : "$(INCLUDE_NAT)") $(info "INCLUDE_DHCP_RELAY" : "$(INCLUDE_DHCP_RELAY)") $(info "INCLUDE_P4RT" : "$(INCLUDE_P4RT)") $(info "INCLUDE_KUBERNETES" : "$(INCLUDE_KUBERNETES)") +$(info "INCLUDE_KUBERNETES_MASTER" : "$(INCLUDE_KUBERNETES_MASTER)") $(info "INCLUDE_MACSEC" : "$(INCLUDE_MACSEC)") $(info "INCLUDE_MUX" : "$(INCLUDE_MUX)") $(info "INCLUDE_BOOTCHART : "$(INCLUDE_BOOTCHART)") @@ -1188,6 +1189,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export shutdown_bgp_on_start="$(SHUTDOWN_BGP_ON_START)" export default_buffer_model="$(SONIC_BUFFER_MODEL)" export include_kubernetes="$(INCLUDE_KUBERNETES)" + export include_kubernetes_master="$(INCLUDE_KUBERNETES_MASTER)" export kube_docker_proxy="$(KUBE_DOCKER_PROXY)" export enable_pfcwd_on_start="$(ENABLE_PFCWD_ON_START)" export installer_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($*_INSTALLS) $(FIPS_BASEIMAGE_INSTALLERS))" @@ -1358,6 +1360,12 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \ CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \ + MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \ + MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION=$(MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION) \ + MASTER_PAUSE_VERSION=$(MASTER_PAUSE_VERSION) \ + MASTER_COREDNS_VERSION=$(MASTER_COREDNS_VERSION) \ + MASTER_ETCD_VERSION=$(MASTER_ETCD_VERSION) \ + MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \ ./build_debian.sh $(LOG) USERNAME="$(USERNAME)" \