Parallel building of sonic dockers using native dockerd(dood). (#10352)

Currently, the build dockers are created as a user dockers(docker-base-stretch-<user>, etc) that are
specific to each user. But the sonic dockers (docker-database, docker-swss, etc) are
created with a fixed docker name and common to all the users.

    docker-database:latest
    docker-swss:latest

When multiple builds are triggered on the same build server that creates parallel building issue because
all the build jobs are trying to create the same docker with latest tag.
This happens only when sonic dockers are built using native host dockerd for sonic docker image creation.

This patch creates all sonic dockers as user sonic dockers and then, while
saving and loading the user sonic dockers, it rename the user sonic
dockers into correct sonic dockers with tag as latest.

	docker-database:latest <== SAVE/LOAD ==> docker-database-<user>:tag

The user sonic docker names are derived from 'DOCKER_USERNAME and DOCKER_USERTAG' make env
variable and using Jinja template, it replaces the FROM docker name with correct user sonic docker name for
loading and saving the docker image.

Makefile

@@ -40,6 +40,7 @@ endif
 ifeq ($(NOBULLSEYE), 0)
 	BLDENV=bullseye make -f Makefile.work $@
 endif
+	BLDENV=bullseye make -f Makefile.work docker-cleanup
 
 jessie:
 	@echo "+++ Making $@ +++"
@@ -83,7 +84,7 @@ $(PLATFORM_PATH):
 configure : $(PLATFORM_PATH)
 	$(call make_work, $@)
 
-clean reset showtag sonic-slave-build sonic-slave-bash :
+clean reset showtag docker-cleanup sonic-slave-build sonic-slave-bash :
 	$(call make_work, $@)
 
 # Freeze the versions, see more detail options: scripts/versions_manager.py freeze -h

Makefile.work

@@ -90,6 +90,7 @@ $(shell rm -f .screen)
 MAKEFLAGS += -B
 
 CONFIGURED_ARCH := $(shell [ -f .arch ] && cat .arch || echo $(PLATFORM_ARCH))
+CONFIGURED_PLATFORM = $(if $(PLATFORM),$(PLATFORM),$(shell cat .platform 2>/dev/null))
 ifeq ($(CONFIGURED_ARCH),)
     override CONFIGURED_ARCH = amd64
 endif
@@ -149,7 +150,9 @@ $(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) s
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}')
+# Calculate the slave TAG based on $(USER)/$(PWD)/$(CONFIGURED_PLATFORM) to get unique SHA ID
+SLAVE_TAG = $(shell (cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* .git/HEAD && echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
+			| sha1sum | awk '{print substr($$1,0,11);}')
 
 OVERLAY_MODULE_CHECK := \
     lsmod | grep -q "^overlay " &>/dev/null || \
@@ -159,6 +162,14 @@ OVERLAY_MODULE_CHECK := \
 
 BUILD_TIMESTAMP := $(shell date +%Y%m%d\.%H%M%S)
 
+# Create separate Docker lockfiles for saving vs. loading an image.
+ifeq ($(DOCKER_LOCKDIR),)
+override DOCKER_LOCKDIR := /tmp/docklock
+endif
+DOCKER_LOCKFILE_SAVE := $(DOCKER_LOCKDIR)/docker_save.lock
+$(shell mkdir -m 0777 -p $(DOCKER_LOCKDIR))
+$(shell [ -f $(DOCKER_LOCKFILE_SAVE) ] || (touch $(DOCKER_LOCKFILE_SAVE) && chmod 0777 $(DOCKER_LOCKFILE_SAVE)))
+
 ifeq ($(DOCKER_BUILDER_MOUNT),)
 override DOCKER_BUILDER_MOUNT := "$(PWD):/sonic"
 endif
@@ -169,6 +180,7 @@ endif
 
 DOCKER_RUN := docker run --rm=true --privileged --init \
     -v $(DOCKER_BUILDER_MOUNT) \
+    -v "$(DOCKER_LOCKDIR):$(DOCKER_LOCKDIR)" \
     -w $(DOCKER_BUILDER_WORKDIR) \
     -e "http_proxy=$(http_proxy)" \
     -e "https_proxy=$(https_proxy)" \
@@ -199,6 +211,30 @@ ifneq ($(SIGNING_CERT),)
 endif
 endif
 
+# User name and tag for "docker-*" images created by native dockerd mode.
+ifeq ($(strip $(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD)),y)
+DOCKER_USERNAME = $(USER_LC)
+DOCKER_USERTAG = $(SLAVE_TAG)
+else
+DOCKER_USERNAME = sonic
+DOCKER_USERTAG = latest
+endif
+
+# Define canned sequence to clean up Docker image cache.
+#   - These are the remnants from building the runtime Docker images using native (host) Docker daemon.
+#   - Image naming convention differs on a shared build system vs. non-shared.
+# $(docker-image-cleanup)
+ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD),y)
+define docker-image-cleanup
+    @for i in $(shell docker images --quiet --filter 'dangling=true') ; do (docker rmi -f $$i &> /dev/null || true) ; done
+    @for i in $(shell docker images --quiet docker-*$(DOCKER_USERNAME):$(DOCKER_USERTAG)) ; do (docker rmi -f $$i &> /dev/null || true) ; done
+endef
+else
+define docker-image-cleanup
+    @:
+endef
+endif
+
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
 ifneq ($(MULTIARCH_QEMU_ENVIRON), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
@@ -274,6 +310,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            BUILD_NUMBER=$(BUILD_NUMBER) \
                            BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) \
                            SONIC_IMAGE_VERSION=$(SONIC_IMAGE_VERSION) \
+                           SLAVE_TAG=$(SLAVE_TAG) \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
                            ENABLE_ZTP=$(ENABLE_ZTP) \
                            INCLUDE_PDE=$(INCLUDE_PDE) \
@@ -298,6 +335,11 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
                            NO_PROXY=$(no_proxy) \
+                           DOCKER_USERNAME=$(DOCKER_USERNAME) \
+                           DOCKER_USERTAG=$(DOCKER_USERTAG) \
+                           DOCKER_LOCKDIR=$(DOCKER_LOCKDIR) \
+                           DOCKER_LOCKFILE_SAVE=$(DOCKER_LOCKFILE_SAVE) \
+                           SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD=$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            INCLUDE_DHCP_RELAY=$(INCLUDE_DHCP_RELAY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
@@ -352,6 +394,9 @@ else
 	@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?"
 endif
 
+docker-cleanup:
+	$(docker-image-cleanup)
+
 sonic-build-hooks:
 	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
 	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo

dockers/docker-config-engine-bullseye/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-bullseye
+FROM docker-base-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-buster
+FROM docker-base-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-config-engine-stretch/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-stretch
+FROM docker-base-stretch:-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-config-engine/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-base
+FROM docker-base-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-database/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-dhcp-relay/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-fpm-frr/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-swss-layer-buster
+FROM docker-swss-layer-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG frr_user_uid

dockers/docker-fpm-gobgp/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-fpm-quagga
+FROM docker-fpm-quagga-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-iccpd/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-lldp/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-macsec/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-mux/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-nat/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, copy_files %}
-FROM docker-swss-layer-buster
+FROM docker-swss-layer-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-orchagent/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-swss-layer-buster
+FROM docker-swss-layer-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-pde/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ docker_pde_load_image }}
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ENV PYTHONPATH=/usr/share/sonic/platform

dockers/docker-platform-monitor/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-ptf-sai/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-ptf
+FROM docker-ptf-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-router-advertiser/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-sflow/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-swss-layer-buster
+FROM docker-swss-layer-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-snmp/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python3_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-sonic-mgmt-framework/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-sonic-p4rt/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG git_commit

dockers/docker-sonic-restapi/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-sonic-sdk-buildenv/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM sonic-sdk
+FROM sonic-sdk-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG base_os_version
 ARG docker_database_version

dockers/docker-sonic-sdk/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-sonic-telemetry/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bullseye
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-swss-layer-buster/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-teamd/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-swss-layer-buster
+FROM docker-swss-layer-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/barefoot/docker-syncd-bfn-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-bfn
+FROM docker-syncd-bfn-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/barefoot/docker-syncd-bfn/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/broadcom/docker-saiserver-brcm/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/broadcom/docker-syncd-brcm-dnx-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-brcm-dnx
+FROM docker-syncd-brcm-dnx-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/broadcom/docker-syncd-brcm-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-brcm
+FROM docker-syncd-brcm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/broadcom/docker-syncd-brcm/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/cavium/docker-syncd-cavm-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-cavm
+FROM docker-syncd-cavm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/cavium/docker-syncd-cavm/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine
+FROM docker-config-engine-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/centec-arm64/docker-saiserver-centec/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/centec-arm64/docker-syncd-centec-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-centec
+FROM docker-syncd-centec-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/centec-arm64/docker-syncd-centec/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/centec/docker-saiserver-centec/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/centec/docker-syncd-centec-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-centec
+FROM docker-syncd-centec-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/centec/docker-syncd-centec/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/components/docker-gbsyncd-credo/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/innovium/docker-syncd-invm-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-invm
+FROM docker-syncd-invm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/innovium/docker-syncd-invm/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-stretch
+FROM docker-config-engine-stretch-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/marvell-arm64/docker-syncd-mrvl-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-mrvl
+FROM docker-syncd-mrvl-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/marvell-arm64/docker-syncd-mrvl/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/marvell-armhf/docker-syncd-mrvl-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-mrvl
+FROM docker-syncd-mrvl-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/marvell-armhf/docker-syncd-mrvl/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/marvell/docker-syncd-mrvl-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-mrvl
+FROM docker-syncd-mrvl-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/marvell/docker-syncd-mrvl/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/mellanox/docker-saiserver-mlnx/Dockerfile.j2

@@ -15,7 +15,7 @@
 ## limitations under the License.
 ##
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/mellanox/docker-syncd-mlnx-rpc/Dockerfile.j2

@@ -15,7 +15,7 @@
 ## limitations under the License.
 ##
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-syncd-mlnx
+FROM docker-syncd-mlnx-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/mellanox/docker-syncd-mlnx/Dockerfile.j2

@@ -15,7 +15,7 @@
 ## limitations under the License.
 ##
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/nephos/docker-syncd-nephos-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-nephos
+FROM docker-syncd-nephos-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/nephos/docker-syncd-nephos/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-stretch
+FROM docker-config-engine-stretch-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/p4/docker-sonic-p4/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine
+FROM docker-config-engine-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/vs/docker-gbsyncd-vs/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/vs/docker-syncd-vs/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM docker-config-engine-buster-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
 
 ARG docker_container_name
 

rules/config

@@ -29,7 +29,10 @@ DEFAULT_BUILD_LOG_TIMESTAMP = none
 # SONIC_USE_DOCKER_BUILDKIT = y
 
 # SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD - use native dockerd for build.
-# If set to y SONiC build container will use native dockerd instead of dind for faster build
+# If set to y SONiC build container will use native dockerd instead of dind for faster build.
+# Special handling of the docker image file names is needed to avoid conflicts with
+# other SONiC build jobs on the same server. This requires changes to the Dockerfile.j2 FROM statement
+# in the dockers/ and platform/ subdirs to use a variable reference instead of an explicit image name.
 # SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD = y
 
 # SONIC_CONFIG_ENABLE_COLORS - enable colored output in build system.

slave.mk

@@ -241,6 +241,10 @@ ifeq ($(SONIC_BUILD_JOBS),)
 override SONIC_BUILD_JOBS := $(SONIC_CONFIG_BUILD_JOBS)
 endif
 
+DOCKER_IMAGE_REF = $*-$(DOCKER_USERNAME):$(DOCKER_USERTAG)
+DOCKER_DBG_IMAGE_REF = $*-$(DBG_IMAGE_MARK)-$(DOCKER_USERNAME):$(DOCKER_USERTAG)
+export DOCKER_USERNAME DOCKER_USERTAG 
+
 ifeq ($(VS_PREPARE_MEM),)
 override VS_PREPARE_MEM := $(DEFAULT_VS_PREPARE_MEM)
 endif
@@ -286,6 +290,7 @@ $(info "CONFIGURED_ARCH"                 : "$(if $(PLATFORM_ARCH),$(PLATFORM_ARC
 $(info "SONIC_CONFIG_PRINT_DEPENDENCIES" : "$(SONIC_CONFIG_PRINT_DEPENDENCIES)")
 $(info "SONIC_BUILD_JOBS"                : "$(SONIC_BUILD_JOBS)")
 $(info "SONIC_CONFIG_MAKE_JOBS"          : "$(SONIC_CONFIG_MAKE_JOBS)")
+$(info "USE_NATIVE_DOCKERD_FOR_BUILD"    : "$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD)")
 $(info "SONIC_USE_DOCKER_BUILDKIT"       : "$(SONIC_USE_DOCKER_BUILDKIT)")
 $(info "USERNAME"                        : "$(USERNAME)")
 $(info "PASSWORD"                        : "$(PASSWORD)")
@@ -361,6 +366,60 @@ endif
 export kernel_procure_method=$(KERNEL_PROCURE_METHOD)
 export vs_build_prepare_mem=$(VS_PREPARE_MEM)
 
+###############################################################################
+## Canned sequences
+###############################################################################
+## When multiple builds are triggered on the same build server that causes the docker image naming problem because
+## all the build jobs are trying to create the same docker image with latest as tag.
+## This happens only when sonic docker images are built using native host dockerd.
+##
+##     docker-swss:latest <=SAVE/LOAD=> docker-swss-<user>:<tag>
+
+# $(call docker-image-save,from,to)
+# Sonic docker images are always created with username as extension. During the save operation, 
+# it removes the username extension from docker image and saved them as compressed tar file for SONiC image generation.
+# The save operation is protected with lock for parallel build.
+#
+# $(1) => Docker name
+# $(2) => Docker target name
+
+define docker-image-save
+    @echo "Attempting docker image lock for $(1) save" $(LOG)
+    $(call MOD_LOCK,$(1),$(DOCKER_LOCKDIR),$(DOCKER_LOCKFILE_SUFFIX),$(DOCKER_LOCKFILE_TIMEOUT))
+    @echo "Obtained docker image lock for $(1) save" $(LOG)
+    @echo "Tagging docker image $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) as $(1):latest" $(LOG)
+    docker tag $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) $(1):latest $(LOG)
+    @echo "Saving docker image $(1):latest" $(LOG)
+        docker save $(1):latest | gzip -c > $(2)
+    @echo "Removing docker image $(1):latest" $(LOG)
+    docker rmi -f $(1):latest $(LOG)
+    $(call MOD_UNLOCK,$(1))
+    @echo "Released docker image lock for $(1) save" $(LOG)
+    @echo "Removing docker image $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG)" $(LOG)
+    docker rmi -f $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) $(LOG)
+endef
+
+# $(call docker-image-load,from)
+# Sonic docker images are always created with username as extension. During the load operation, 
+# it loads the docker image from compressed tar file and tag them with username as extension.
+# The load operation is protected with lock for parallel build.
+#
+# $(1) => Docker name
+# $(2) => Docker target name
+define docker-image-load
+    @echo "Attempting docker image lock for $(1) load" $(LOG)
+    $(call MOD_LOCK,$(1),$(DOCKER_LOCKDIR),$(DOCKER_LOCKFILE_SUFFIX),$(DOCKER_LOCKFILE_TIMEOUT))
+    @echo "Obtained docker image lock for $(1) load" $(LOG)
+    @echo "Loading docker image $(TARGET_PATH)/$(1).gz" $(LOG)
+    docker load -i $(TARGET_PATH)/$(1).gz $(LOG)
+    @echo "Tagging docker image $(1):latest as $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG)" $(LOG)
+    docker tag $(1):latest $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) $(LOG)
+    @echo "Removing docker image $(1):latest" $(LOG)
+    docker rmi -f $(1):latest $(LOG)
+    $(call MOD_UNLOCK,$(1))
+    @echo "Released docker image lock for $(1) load" $(LOG)
+endef
+
 ###############################################################################
 ## Local targets
 ###############################################################################
@@ -752,9 +811,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
 		--build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \
 		--label Tag=$(SONIC_IMAGE_VERSION) \
 		-f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \
-		-t $* $($*.gz_PATH) $(LOG)
+		-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
-	scripts/collect_docker_version_files.sh $* $(TARGET_PATH)
+	scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
-	docker save $* | gzip -c > $@
+	$(call docker-image-save,$*,$@)
 	# Clean up
 	if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
 	$(FOOTER)
@@ -871,9 +930,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 			--label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 		        $($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \
-			-t $* $($*.gz_PATH) $(LOG)
+			-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
-		scripts/collect_docker_version_files.sh $* $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
-		docker save $* | gzip -c > $@
+		$(call docker-image-save,$*,$@)
 		# Clean up
 		if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
 
@@ -885,7 +944,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 
 SONIC_TARGET_LIST += $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES))
 
-# Targets for building docker images
+# Targets for building docker debug images
 $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAGE_MARK).gz : .platform docker-start \
 		$$(addprefix $(TARGET_PATH)/,$$($$*.gz_AFTER)) \
 		$$(addprefix $$($$*.gz_DEBS_PATH)/,$$($$*.gz_DBG_DEPENDS)) \
@@ -905,7 +964,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++'))
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_pkgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_APT_PACKAGES),RDEPENDS))\n" | awk '!a[$$0]++'))
-		./build_debug_docker_j2.sh $* $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs > $($*.gz_PATH)/Dockerfile-dbg.j2
+		./build_debug_docker_j2.sh $(DOCKER_IMAGE_REF) $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs > $($*.gz_PATH)/Dockerfile-dbg.j2
 		j2 $($*.gz_PATH)/Dockerfile-dbg.j2 > $($*.gz_PATH)/Dockerfile-dbg
 		$(call generate_manifest,$*,dbg)
 		# Prepare docker build info
@@ -923,10 +982,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 			--label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 			--file $($*.gz_PATH)/Dockerfile-dbg \
-			-t $*-dbg $($*.gz_PATH) $(LOG)
+			-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
-		scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH)
-		docker save $*-dbg | gzip -c > $@
+		$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
 		# Clean up
+		docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true
 		if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
 
 		# Save the target deb into DPKG cache
@@ -952,7 +1012,7 @@ endif
 
 $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TARGET_PATH)/$$*.gz
 	$(HEADER)
-	docker load -i $(TARGET_PATH)/$*.gz $(LOG)
+	$(call docker-image-load,$*)
 	$(FOOTER)
 
 ###############################################################################