diff --git a/Makefile.work b/Makefile.work index 4c73dbfc88..d751f18fce 100644 --- a/Makefile.work +++ b/Makefile.work @@ -53,6 +53,9 @@ # * ENABLE_BOOTCHART: Enable SONiC bootchart # * Default: n # * Values: y,n +# * GZ_COMPRESS_PROGRAM: Select pigz to reduce build time +# * Default: gzip +# * Values: pigz,gzip # ############################################################################### @@ -140,6 +143,10 @@ ifeq ($(ENABLE_DOCKER_BASE_PULL),) override ENABLE_DOCKER_BASE_PULL = n endif +ifneq ($(GZ_COMPRESS_PROGRAM), pigz) +override GZ_COMPRESS_PROGRAM = gzip +endif + ifeq ($(CONFIGURED_ARCH),amd64) SLAVE_BASE_IMAGE = $(SLAVE_DIR) MULTIARCH_QEMU_ENVIRON = n @@ -200,6 +207,7 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \ ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \ DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) \ DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ + GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \ j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile) $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \ @@ -525,6 +533,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \ MIRROR_URLS=$(MIRROR_URLS) \ MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \ + GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset diff --git a/build_debian.sh b/build_debian.sh index f5fb7409f4..3668e62733 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -138,6 +138,12 @@ if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then else sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c 'cd /dev && MAKEDEV generic' fi + +## docker and mkinitramfs on target system will use pigz/unpigz automatically +if [[ $GZ_COMPRESS_PROGRAM == pigz ]]; then + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install pigz +fi + ## Install initramfs-tools and linux kernel ## Note: initramfs-tools recommends depending on busybox, and we really want busybox for ## 1. commands such as touch @@ -699,8 +705,8 @@ if [[ $MULTIARCH_QEMU_ENVIRON == y || $CROSS_BUILD_ENVIRON == y ]]; then fi ## Compress docker files -pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd +pushd $FILESYSTEM_ROOT && sudo tar -I $GZ_COMPRESS_PROGRAM -cf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo tar czf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd +pushd $FILESYSTEM_ROOT && sudo tar -I $GZ_COMPRESS_PROGRAM -cf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd sudo zip -g -n .squashfs:.gz $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS diff --git a/build_docker.sh b/build_docker.sh index ec07698d66..9ada37d39b 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -101,7 +101,8 @@ fi ## Save the docker image in a gz file mkdir -p target -docker save $docker_image_name | gzip -c > target/$docker_image_name.gz +command -v pigz > /dev/null && GZ_COMPRESS_PROGRAM=pigz || GZ_COMPRESS_PROGRAM=gzip +docker save $docker_image_name | $GZ_COMPRESS_PROGRAM -c > target/$docker_image_name.gz if [ -n "$1" ]; then ./push_docker.sh target/$docker_image_name.gz $@ $docker_image_tag diff --git a/build_image.sh b/build_image.sh index ea276367eb..847689702f 100755 --- a/build_image.sh +++ b/build_image.sh @@ -56,10 +56,10 @@ generate_kvm_image() exit 1 } - gzip $KVM_IMAGE_DISK + $GZ_COMPRESS_PROGRAM $KVM_IMAGE_DISK [ -r $KVM_IMAGE_DISK.gz ] || { - echo "Error : gzip $KVM_IMAGE_DISK failed!" + echo "Error : $GZ_COMPRESS_PROGRAM $KVM_IMAGE_DISK failed!" exit 1 } @@ -147,10 +147,10 @@ elif [ "$IMAGE_TYPE" = "raw" ]; then exit 1 } - gzip $OUTPUT_RAW_IMAGE + $GZ_COMPRESS_PROGRAM $OUTPUT_RAW_IMAGE [ -r $OUTPUT_RAW_IMAGE.gz ] || { - echo "Error : gzip $OUTPUT_RAW_IMAGE failed!" + echo "Error : $GZ_COMPRESS_PROGRAM $OUTPUT_RAW_IMAGE failed!" exit 1 } diff --git a/rules/config b/rules/config index 69a6cf0ddd..c72c572ab6 100644 --- a/rules/config +++ b/rules/config @@ -275,3 +275,7 @@ ENABLE_FIPS ?= n # SONIC_SLAVE_DOCKER_DRIVER - set the sonic slave docker storage driver SONIC_SLAVE_DOCKER_DRIVER ?= vfs + +# GZ_COMPRESS_PROGRAM - select pigz (a parallel implementation of gzip) to reduce a build time +# and speed up a decompression of docker images on target system +GZ_COMPRESS_PROGRAM ?= gzip diff --git a/slave.mk b/slave.mk index 3460a3e184..3a3381ff2d 100644 --- a/slave.mk +++ b/slave.mk @@ -86,6 +86,7 @@ export DOCKER_BASE_ARCH export CROSS_BUILD_ENVIRON export BLDENV export BUILD_WORKDIR +export GZ_COMPRESS_PROGRAM ############################################################################### ## Utility rules @@ -431,6 +432,7 @@ ifeq ($(CONFIGURED_PLATFORM),vs) $(info "BUILD_MULTIASIC_KVM" : "$(BUILD_MULTIASIC_KVM)") endif $(info "CROSS_BUILD_ENVIRON" : "$(CROSS_BUILD_ENVIRON)") +$(info "GZ_COMPRESS_PROGRAM" : "$(GZ_COMPRESS_PROGRAM)") $(info ) else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) @@ -489,7 +491,7 @@ define docker-image-save @echo "Tagging docker image $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) as $(1):$(call docker-get-tag,$(1))" $(LOG) docker tag $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) $(1):$(call docker-get-tag,$(1)) $(LOG) @echo "Saving docker image $(1):$(call docker-get-tag,$(1))" $(LOG) - docker save $(1):$(call docker-get-tag,$(1)) | gzip -c > $(2) + docker save $(1):$(call docker-get-tag,$(1)) | $(GZ_COMPRESS_PROGRAM) -c > $(2) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then @echo "Removing docker image $(1):$(call docker-get-tag,$(1))" $(LOG) docker rmi -f $(1):$(call docker-get-tag,$(1)) $(LOG) diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index f4f24d847e..7141cfa4c6 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -87,6 +87,7 @@ RUN apt-get update && apt-get install -y \ curl \ wget \ unzip \ + {{ GZ_COMPRESS_PROGRAM }} \ git \ build-essential \ libtool \ diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 0c2ce405f2..35b0a9e843 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -89,6 +89,7 @@ RUN apt-get update && apt-get install -y \ curl \ wget \ unzip \ + {{ GZ_COMPRESS_PROGRAM }} \ git \ build-essential \ libtool \ diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 1397df1c73..641fa69c07 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ curl \ wget \ unzip \ + {{ GZ_COMPRESS_PROGRAM }} \ git \ build-essential \ libtool \