Add k8s master code new (#15716)
Why I did it Currently, k8s master image is generated from a separate branch which we created by ourselves, not release ones. We need to commit these k8s master related code to master branch for a better way to do k8s master image build out. Work item tracking Microsoft ADO (number only): 19998138 How I did it Install k8s dashboard docker images Install geneva mds and mdsd and fluentd docker images and tag them as latest, tagging latest will help create container always with the latest version Install azure-storage-blob and azure-identity, this will help do etcd backup and restore. Install kubernetes python client packages, this will help read worker and container state, we can send these metric to Geneva. Remove mdm debian package, will replace it with the mdm docker image Add k8s master entrance script, this script will be called by rc-local service when system startup. we have some master systemd services in compute-move repo, when VMM service create master VM, VMM will copy all master service files inside VM, the entrance script will setup all services according to the service files. When the entrance script content changed, the PR build will set include_kubernetes_master=y to help do validation for k8s master related code change. The default value of include_kubernetes_master should be always n for public master branch. We will generate master image from internal master branch How to verify it Build with INCLUDE_KUBERNETES_MASTER = y
This commit is contained in:
lixiaoyuner
GitHub
parent
d2b5d774c5
commit
10b65d9826
@ -130,6 +130,10 @@ jobs:
|
||||
make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
|
||||
mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
|
||||
fi
|
||||
if [ $(K8S_OPTIONS) == 'INCLUDE_KUBERNETES_MASTER=y' ]; then
|
||||
make $BUILD_OPTIONS $(K8S_OPTIONS) target/sonic-vs.img.gz
|
||||
mv target/sonic-vs.img.gz target/sonic-vs-k8s.img.gz
|
||||
fi
|
||||
make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
|
||||
make $BUILD_OPTIONS target/docker-ptf-sai.gz
|
||||
if [ $(Build.Reason) != 'PullRequest' ];then
|
||||
|
||||
@ -3,6 +3,13 @@ steps:
|
||||
- script: |
|
||||
set -ex
|
||||
tar_branch=origin/$(System.PullRequest.TargetBranch)
|
||||
# Check if k8s master entrance script is changed
|
||||
k8s_master_changed=$(git diff $tar_branch..HEAD --name-only | grep -F files/image_config/kubernetes/kubernetes_master_entrance.sh)
|
||||
if [ -z "$k8s_master_changed" ]; then
|
||||
echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=n"
|
||||
else
|
||||
echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=y"
|
||||
fi
|
||||
git diff $tar_branch..HEAD --name-only | grep -v -f .azure-pipelines/vstest-exclude && exit 0
|
||||
git diff $tar_branch..HEAD --name-only | grep -f .azure-pipelines/vstest-include && exit 0
|
||||
set +x
|
||||
|
||||
@ -295,17 +295,8 @@ then
|
||||
echo '[INFO] Install kubernetes master'
|
||||
install_kubernetes ${MASTER_KUBERNETES_VERSION}
|
||||
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
|
||||
https://packages.microsoft.com/keys/microsoft.asc | \
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
|
||||
https://packages.microsoft.com/keys/msopentech.asc | \
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
|
||||
echo "deb [arch=amd64] https://packages.microsoft.com/repos/azurecore-debian $IMAGE_DISTRO main" | \
|
||||
sudo tee $FILESYSTEM_ROOT/etc/apt/sources.list.d/azure.list
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install metricsext2
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \
|
||||
https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb
|
||||
|
||||
@ -790,15 +790,27 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIV
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-proxy:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/coredns/coredns:${MASTER_COREDNS_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/etcd:${MASTER_ETCD_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/metrics-scraper:${MASTER_UI_METRIC_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/dashboard:${MASTER_UI_DASH_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:latest
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:latest
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION}
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:latest
|
||||
echo "kubernetes master docker images pull complete"
|
||||
# Install python package for mdm service usage
|
||||
# Install python packages for mdm metrics collection service usage
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install statsd
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable mdm.service
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install kubernetes
|
||||
# Install python packages to upload and download etcd backup files for backup and restore service usage
|
||||
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-storage-blob azure-identity
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubelet.service
|
||||
# Add kubernetes master entrance
|
||||
sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/sbin/
|
||||
sudo sed -i '/^exit 0/i\bash /usr/sbin/kubernetes_master_entrance.sh' $FILESYSTEM_ROOT/etc/rc.local
|
||||
sudo cp files/image_config/kubernetes/kubernetes_master_entrance.service ${FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM}/
|
||||
sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/bin/
|
||||
sudo chmod +x $FILESYSTEM_ROOT/usr/bin/kubernetes_master_entrance.sh
|
||||
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubernetes_master_entrance.service
|
||||
{% endif %}
|
||||
|
||||
{% macro get_install_options(set_owner, enabled) -%}
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Setup k8s master services
|
||||
Requires=rc-local.service
|
||||
After=rc-local.service
|
||||
Before=sonic.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/bin/kubernetes_master_entrance.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,2 +1,47 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script is for kubernetes master image usage
|
||||
# Will mount kubernetes master disk and execute kubernetes entrance script
|
||||
|
||||
set -x
|
||||
|
||||
# mount disk from host
|
||||
mount_point="/from_host"
|
||||
disk_by_path_dir="/dev/disk/by-path"
|
||||
# We can't ensure the mounted disk name is sda or sdb
|
||||
# Currently we specify the disk logic unit number to 2 when create master VM
|
||||
# We find the correct disk by the disk logic unit number from by-path directory
|
||||
target_str="lun-2-part1"
|
||||
disk_path_file=$(ls $disk_by_path_dir | grep $target_str)
|
||||
|
||||
# Check whether the disk path file exists
|
||||
if [ -z "$disk_path_file" ]; then
|
||||
echo "Error: Disk path file not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check the number of lines returned
|
||||
line_count=$(echo "$disk_path_file" | wc -l)
|
||||
|
||||
# If there are multiple lines, exit with status 1
|
||||
if [ "$line_count" -ne 1 ]; then
|
||||
echo "Error: multiple disk path files found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
disk="${disk_by_path_dir}/${disk_path_file}"
|
||||
mkdir -p $mount_point
|
||||
mount $disk $mount_point
|
||||
|
||||
# check whether it is the first time to boot
|
||||
first_boot_flag_file="/from_host/first_boot_flag"
|
||||
if [ -f $first_boot_flag_file ]; then
|
||||
exit 0
|
||||
fi
|
||||
touch $first_boot_flag_file
|
||||
|
||||
# execute entrance script
|
||||
init_file_name="entrance.sh"
|
||||
init_file=${mount_point}/${init_file_name}
|
||||
chmod +x $init_file
|
||||
source $init_file
|
||||
|
||||
13
rules/config
13
rules/config
@ -199,12 +199,23 @@ INCLUDE_KUBERNETES_MASTER ?= n
|
||||
# MASTER_PAUSE_VERSION - version of pause container image
|
||||
# MASTER_COREDNS_VERSION - version of coredns container image
|
||||
# MASTER_ETCD_VERSION = version of etcd container image
|
||||
# MASTER_CRI_DOCKERD = version of cri-dockerd container image
|
||||
# MASTER_UI_METRIC_VERSION = version of k8s metrics server container image
|
||||
# MASTER_UI_DASH_VERSION = version of k8s dashboard container image
|
||||
# MASTER_MDM_VERSION = version of mdm container image
|
||||
# MASTER_MDS_VERSION = version of mds container image
|
||||
# MASTER_FLUENTD_VERSION = version of fluentd container image
|
||||
MASTER_KUBERNETES_VERSION = 1.22.2-00
|
||||
MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION = v1.22.2
|
||||
MASTER_PAUSE_VERSION = 3.5
|
||||
MASTER_COREDNS_VERSION = v1.8.4
|
||||
MASTER_ETCD_VERSION = 3.5.0-0
|
||||
MASTER_CRI_DOCKERD = 0.2.5
|
||||
MASTER_CRI_DOCKERD = 0.3.1
|
||||
MASTER_UI_METRIC_VERSION = v1.0.8
|
||||
MASTER_UI_DASH_VERSION = v2.7.0
|
||||
MASTER_MDM_VERSION = 2.2023.505.1124-45da18-20230505t1700
|
||||
MASTER_MDS_VERSION = mariner_20230517.1
|
||||
MASTER_FLUENTD_VERSION = mariner_20230517.1
|
||||
|
||||
# SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
|
||||
# To not use the auto-generated self-signed certificate, the required files to sign the image as below:
|
||||
|
||||
5
slave.mk
5
slave.mk
@ -1448,6 +1448,11 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
|
||||
MASTER_COREDNS_VERSION=$(MASTER_COREDNS_VERSION) \
|
||||
MASTER_ETCD_VERSION=$(MASTER_ETCD_VERSION) \
|
||||
MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
|
||||
MASTER_UI_METRIC_VERSION=$(MASTER_UI_METRIC_VERSION) \
|
||||
MASTER_UI_DASH_VERSION=$(MASTER_UI_DASH_VERSION) \
|
||||
MASTER_MDM_VERSION=$(MASTER_MDM_VERSION) \
|
||||
MASTER_MDS_VERSION=$(MASTER_MDS_VERSION) \
|
||||
MASTER_FLUENTD_VERSION=$(MASTER_FLUENTD_VERSION) \
|
||||
./build_debian.sh $(LOG)
|
||||
|
||||
USERNAME="$(USERNAME)" \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user