[Build] Improve docker build performance (#11111)

Why I did it
The docker storage driver vfs is not a good option for build, it uses the “deep copy” when building a new layer, leads to lower performance and more space used on disk than other storage drivers.
A better docker storage driver is the default one overlay2, it is a modern union filesystem.

.azure-pipelines/template-variables.yml

@@ -1,3 +1,4 @@
 variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
   COMMON_LIB_BUILD_ENVS: 'bullseye'
+  SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'

Makefile.work

@@ -137,6 +137,7 @@ MULTIARCH_QEMU_ENVIRON = y
 endif
 endif
 SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
+DOCKER_ROOT = $(PWD)/fsroot.docker.$(BLDENV)
 
 # Support FIPS feature, armhf not supported yet
 ifeq ($(PLATFORM_ARCH),armhf)
@@ -181,6 +182,7 @@ endif
 DOCKER_LOCKFILE_SAVE := $(DOCKER_LOCKDIR)/docker_save.lock
 $(shell mkdir -m 0777 -p $(DOCKER_LOCKDIR))
 $(shell [ -f $(DOCKER_LOCKFILE_SAVE) ] || (touch $(DOCKER_LOCKFILE_SAVE) && chmod 0777 $(DOCKER_LOCKFILE_SAVE)))
+$(shell sudo rm -rf $(DOCKER_ROOT) && mkdir -p $(DOCKER_ROOT))
 
 ifeq ($(DOCKER_BUILDER_MOUNT),)
 override DOCKER_BUILDER_MOUNT := "$(PWD):/sonic"
@@ -200,6 +202,11 @@ DOCKER_RUN := docker run --rm=true --privileged --init \
     -i$(shell { if [ -t 0 ]; then echo t; fi }) \
     $(SONIC_BUILDER_EXTRA_CMDLINE)
 
+# Mount the $(DOCKER_ROOT) to /var/lib/docker in the slave container, the overlay fs is not supported as dockerd root folder.
+ifneq ($(filter $(SONIC_SLAVE_DOCKER_DRIVER),overlay overlay2),)
+	DOCKER_RUN += -v $(DOCKER_ROOT):/var/lib/docker
+endif
+
 ifneq ($(DOCKER_BUILDER_USER_MOUNT),)
 	DOCKER_RUN += $(foreach mount,$(subst $(comma), ,$(DOCKER_BUILDER_USER_MOUNT)), $(addprefix -v , $(mount)))
 endif
@@ -376,6 +383,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            ENABLE_ASAN=$(ENABLE_ASAN) \
                            ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \
                            ENABLE_FIPS=$(ENABLE_FIPS) \
+                           SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

rules/config

@@ -229,3 +229,6 @@ DEFAULT_CONTAINER_REGISTRY ?=
 # ENABLE_FIPS - support FIPS flag, if enabled, no additional config requred for the image to support FIPS
 ENABLE_FIPS_FEATURE ?= y
 ENABLE_FIPS ?= n
+
+# SONIC_SLAVE_DOCKER_DRIVER - set the sonic slave docker storage driver
+SONIC_SLAVE_DOCKER_DRIVER ?= vfs

slave.mk

@@ -791,6 +791,7 @@ $(SONIC_INSTALL_WHEELS) : $(PYTHON_WHEELS_PATH)/%-install : .platform $$(addsuff
 
 # start docker daemon
 docker-start :
+	@sudo sed -i 's/--storage-driver=vfs/--storage-driver=$(SONIC_SLAVE_DOCKER_DRIVER)/' /etc/default/docker
 	@sudo sed -i -e '/http_proxy/d' -e '/https_proxy/d' /etc/default/docker
 	@sudo bash -c "{ echo \"export http_proxy=$$http_proxy\"; \
 	            echo \"export https_proxy=$$https_proxy\"; \