matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[build] add support for 2 stage rootfs build (#15924)

Browse Source 
This adds optimization for the SONiC image build by splitting the final build step into two stages. It allows running the first stage in parallel, improving build time.

The optimization is enabled via new rules/config flag ENABLE_RFS_SPLIT_BUILD (disabled by default)

- Why I did it
To improve a build time.

- How I did it
Added a logic to run build_debian.sh in two stages, transferring the progress via a new build artifact.

- How to verify it
make ENABLE_RFS_SPLIT_BUILD=y SONIC_BUILD_JOBS=32 target/<IMAGE_NAME>.bin

Signed-off-by: Yakiv Huryk <yhuryk@nvidia.com>
This commit is contained in:
Yakiv Huryk 2023-10-11 09:33:17 +03:00 committed by GitHub
parent 7059f42385
commit 6cb8893180
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 177 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Makefile.cache
View File

@ -357,6 +357,21 @@ define SAVE_CACHE
$(if $(call CHECK_WCACHE_ENABLED,$(1)), $(call SAVE_INTO_CACHE,$(1),$(2)))
endef
RFS_DEP_FILES := $(wildcard \
$(addprefix scripts/, build_debian_base_system.sh prepare_debian_image_buildinfo.sh build_mirror_config.sh) \
$(shell git ls-files files/initramfs-tools) \
$(shell git ls-files files/image_config) \
$(shell git ls-files files/apparmor) \
$(shell git ls-files files/apt) \
$(shell git ls-files files/sshd) \
$(shell git ls-files files/dhcp) \
src/sonic-build-hooks/buildinfo/trusted.gpg.d \
platform/$(CONFIGURED_PLATFORM)/modules \
files/docker/docker.service.conf \
files/build_templates/default_users.json.j2 \
files/build_scripts/generate_asic_config_checksum.py \
files/scripts/core_cleanup.py \
build_debian.sh onie-image.conf)
# Set the target path for each target.
@ -384,11 +399,17 @@ $(foreach pkg, $(SONIC_INSTALL_PKGS), \
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FSROOT_PATH))) \
$(eval $(FSROOT_PATH)/$(pkg)_TARGET := $(pkg)) )
$(foreach pkg, $(SONIC_RFS_TARGETS), \
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(TARGET_PATH))) \
$(eval $(pkg)_CACHE_MODE := GIT_CONTENT_SHA) \
$(eval $(pkg)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)) \
$(eval $(pkg)_DEP_FILES := $(SONIC_COMMON_BASE_FILES_LIST) $(RFS_DEP_FILES)) \
$(eval $(TARGET_PATH)/$(pkg)_TARGET := $(pkg)) )
# define the DEP files(.dep and .smdep) and SHA files (.sha and smsha) for each target
$(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS) \
$(SONIC_MAKE_FILES) $(SONIC_PYTHON_STDEB_DEBS) $(SONIC_PYTHON_WHEELS) \
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS), \
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS) $(SONIC_RFS_TARGETS), \
$(eval $(pkg)_MOD_SRC_PATH:=$(if $($(pkg)_SRC_PATH),$($(pkg)_SRC_PATH),$($(pkg)_PATH))) \
$(eval $(pkg)_BASE_PATH:=$(if $($(pkg)_BASE_PATH),$($(pkg)_BASE_PATH),$(CURDIR))) \
$(eval $(pkg)_DEP_FLAGS_FILE:=$($(pkg)_DST_PATH)/$(pkg).flags) \
@ -489,6 +510,7 @@ $(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),f
$(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),flags))
@ -585,6 +607,7 @@ $(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),dep
$(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),dep))
@ -618,6 +641,7 @@ SONIC_CACHE_CLEAN_TARGETS = $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
$(SONIC_DOCKER_IMAGES) \
$(SONIC_DOCKER_DBG_IMAGES) \
$(SONIC_SIMPLE_DOCKER_IMAGES) \
$(SONIC_RFS_TARGETS) \
$(SONIC_INSTALLERS)))
$(SONIC_CACHE_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \

77
build_debian.sh
View File

@ -59,6 +59,9 @@ TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d
exit 1
}
## Check if not a last stage of RFS build
if [[ $RFS_SPLIT_LAST_STAGE != y ]]; then
## Prepare the file system directory
if [[ -d $FILESYSTEM_ROOT ]]; then
sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory"
@ -71,11 +74,6 @@ touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime
## ensure proc is mounted
sudo mount proc /proc -t proc || true
## make / as a mountpoint in chroot env, needed by dockerd
pushd $FILESYSTEM_ROOT
sudo mount --bind . .
popd
## Build the host debian base system
echo '[INFO] Build host debian base system...'
TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
@ -576,24 +574,11 @@ if [ -f files/image_config/ntp/ntp-systemd-wrapper ]; then
sudo cp ./files/image_config/ntp/ntp-systemd-wrapper $FILESYSTEM_ROOT/usr/lib/ntp/
fi
## Version file
## Version file part 1
sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic
if [ -f files/image_config/sonic_release ]; then
sudo cp files/image_config/sonic_release $FILESYSTEM_ROOT/etc/sonic/
fi
export build_version="${SONIC_IMAGE_VERSION}"
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
export kernel_version="${kversion}"
export asic_type="${sonic_asic_platform}"
export asic_subtype="${TARGET_MACHINE}"
export commit_id="$(git rev-parse --short HEAD)"
export branch="$(git rev-parse --abbrev-ref HEAD)"
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
export build_date="$(date -u)"
export build_number="${BUILD_NUMBER:-0}"
export built_by="$USER@$BUILD_HOSTNAME"
export sonic_os_version="${SONIC_OS_VERSION}"
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml
# Default users info
export password_expire="$( [[ "$CHANGE_DEFAULT_PASSWORD" == "y" ]] && echo true || echo false )"
@ -615,6 +600,60 @@ if [[ ! -f './asic_config_checksum' ]]; then
fi
sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum
## Check if not a last stage of RFS build
fi
if [[ $RFS_SPLIT_FIRST_STAGE == y ]]; then
echo '[INFO] Finished with RFS first stage'
echo '[INFO] Umount all'
## Display all process details access /proc
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -vm /proc
## Kill the processes
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /proc || true
## Wait fuser fully kill the processes
sudo timeout 15s bash -c 'until LANG=C chroot $0 umount /proc; do sleep 1; done' $FILESYSTEM_ROOT || true
sudo rm -f $TARGET_PATH/$RFS_SQUASHFS_NAME
sudo mksquashfs $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME -Xcompression-level 1
exit 0
fi
if [[ $RFS_SPLIT_LAST_STAGE == y ]]; then
echo '[INFO] RFS build: second stage'
## ensure proc is mounted
sudo mount proc /proc -t proc || true
sudo fuser -vm $FILESYSTEM_ROOT || true
sudo rm -rf $FILESYSTEM_ROOT
sudo unsquashfs -d $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME
## make / as a mountpoint in chroot env, needed by dockerd
pushd $FILESYSTEM_ROOT
sudo mount --bind . .
popd
trap_push 'sudo LANG=C chroot $FILESYSTEM_ROOT umount /proc || true'
sudo LANG=C chroot $FILESYSTEM_ROOT mount proc /proc -t proc
fi
## Version file part 2
export build_version="${SONIC_IMAGE_VERSION}"
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
export kernel_version="${kversion}"
export asic_type="${sonic_asic_platform}"
export asic_subtype="${TARGET_MACHINE}"
export commit_id="$(git rev-parse --short HEAD)"
export branch="$(git rev-parse --abbrev-ref HEAD)"
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
export build_date="$(date -u)"
export build_number="${BUILD_NUMBER:-0}"
export built_by="$USER@$BUILD_HOSTNAME"
export sonic_os_version="${SONIC_OS_VERSION}"
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml
if [ -f sonic_debian_extension.sh ]; then
./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR $IMAGE_DISTRO
fi

1
rules/functions
View File

@ -143,7 +143,6 @@ SONIC_DBG_DOCKERS += $(2)
endef
###############################################################################
## Utility functions
###############################################################################

96
slave.mk
View File

@ -448,6 +448,34 @@ else
$(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))
endif
# Definition of SONIC_RFS_TARGETS
define rfs_get_installer_dependencies
$(call rfs_build_target_name,$(1),$($(1)_MACHINE))
endef
define rfs_build_target_name
$(1)__$(2)__rfs.squashfs
endef
define rfs_define_target
$(eval rfs_target=$(call rfs_build_target_name,$(1),$($(1)_MACHINE)))
$(eval $(rfs_target)_INSTALLER=$(1))
$(eval $(rfs_target)_MACHINE=$($(1)_MACHINE))
$(eval SONIC_RFS_TARGETS+=$(rfs_target))
$(if $($(1)_DEPENDENT_MACHINE),\
$(eval dependent_rfs_target=$(call rfs_build_target_name,$(1),$($(1)_DEPENDENT_MACHINE)))
$(eval $(dependent_rfs_target)_INSTALLER=$(1))
$(eval $(dependent_rfs_target)_MACHINE=$($(1)_DEPENDENT_MACHINE))
$(eval SONIC_RFS_TARGETS+=$(dependent_rfs_target))
$(eval $(rfs_target)_DEPENDENT_RFS=$(dependent_rfs_target)))
endef
$(foreach installer,$(SONIC_INSTALLERS),$(eval $(call rfs_define_target,$(installer))))
$(foreach installer, $(SONIC_INSTALLERS), $(eval $(installer)_RFS_DEPENDS=$(call rfs_get_installer_dependencies,$(installer))))
SONIC_TARGET_LIST += $(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS))
# Overwrite the buildinfo in slave container
ifeq ($(filter clean,$(MAKECMDGOALS)),)
$(shell DBGOPT='$(DBGOPT)' scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV))
@ -1210,6 +1238,62 @@ $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TA
## Installers
###############################################################################
$(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS)) : $(TARGET_PATH)/% : \
.platform \
build_debian.sh \
$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL)) \
$(addsuffix -install,$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(DEBOOTSTRAP))) \
$$(addprefix $(TARGET_PATH)/,$$($$*_DEPENDENT_RFS)) \
$(call dpkg_depend,$(TARGET_PATH)/%.dep)
$(HEADER)
$(call LOAD_CACHE,$*,$@)
# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then
$(eval installer=$($*_INSTALLER))
$(eval machine=$($*_MACHINE))
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
export initramfs_tools="$(IMAGE_DISTRO_DEBS_PATH)/$(INITRAMFS_TOOLS)"
export linux_kernel="$(IMAGE_DISTRO_DEBS_PATH)/$(LINUX_KERNEL)"
export kversion="$(KVERSION)"
export image_type="$($(installer)_IMAGE_TYPE)"
export sonicadmin_user="$(USERNAME)"
export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))"
export RFS_SPLIT_FIRST_STAGE=y
export RFS_SPLIT_LAST_STAGE=n
j2 -f env files/initramfs-tools/union-mount.j2 onie-image.conf > files/initramfs-tools/union-mount
j2 -f env files/initramfs-tools/arista-convertfs.j2 onie-image.conf > files/initramfs-tools/arista-convertfs
RFS_SQUASHFS_NAME=$* \
USERNAME="$(USERNAME)" \
PASSWORD="$(PASSWORD)" \
CHANGE_DEFAULT_PASSWORD="$(CHANGE_DEFAULT_PASSWORD)" \
TARGET_MACHINE=$(machine) \
IMAGE_TYPE=$($(installer)_IMAGE_TYPE) \
TARGET_PATH=$(TARGET_PATH) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_ENABLE_SECUREBOOT_SIGNATURE="$(SONIC_ENABLE_SECUREBOOT_SIGNATURE)" \
SIGNING_KEY="$(SIGNING_KEY)" \
SIGNING_CERT="$(SIGNING_CERT)" \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
DBGOPT='$(DBGOPT)' \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \
MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
./build_debian.sh $(LOG)
$(call SAVE_CACHE,$*,$@)
fi
$(FOOTER)
# targets for building installers with base image
$(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
.platform \
@ -1264,7 +1348,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \
$$(addprefix $(TARGET_PATH)/,$$($$*_RFS_DEPENDS))
$(HEADER)
# Pass initramfs and linux kernel explicitly. They are used for all platforms
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@ -1420,6 +1506,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
chmod +x sonic_debian_extension.sh,
)
export RFS_SPLIT_FIRST_STAGE=n
export RFS_SPLIT_LAST_STAGE=y
# Build images for the MACHINE, DEPENDENT_MACHINE defined.
$(foreach dep_machine, $($*_MACHINE) $($*_DEPENDENT_MACHINE), \
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
@ -1427,6 +1516,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
scripts/dbg_files.sh
RFS_SQUASHFS_NAME=$*__$(dep_machine)__rfs.squashfs \
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
USERNAME="$(USERNAME)" \
@ -1521,7 +1611,9 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
$(SONIC_DOCKER_IMAGES) \
$(SONIC_DOCKER_DBG_IMAGES) \
$(SONIC_SIMPLE_DOCKER_IMAGES) \
$(SONIC_INSTALLERS)))
$(SONIC_INSTALLERS) \
$(SONIC_RFS_TARGETS)))
$(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
$(Q)rm -rf $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*)