[ci] Set default ACR in UpgrateVersion/PR/official pipeline. (#11002)

* [ci] Set default ACR in UpgrateVersion/PR/official pipeline. (#10341)

Why I did it
docker hub will limit the pull rate.
Use ACR instead to pull debian related docker image.

How I did it
Set DEFAULT_CONTAINER_REGISTRY in pipeline.

* Add a config variable to override default container registry instead of dockerhub. (#10166)
* Add variable to reset default docker registry
* fix bug in docker version control

.azure-pipelines/official-build.yml

@@ -18,9 +18,20 @@ schedules:
     - 201911
     - 201811
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 trigger: none
 pr: none
 
+variables:
+- template: .azure-pipelines/template-variables.yml@buildimage
+
 stages:
 - stage: Build
   pool: sonicbld

Makefile.work

@@ -79,7 +79,7 @@ SLAVE_DIR = sonic-slave-stretch
 else
 SLAVE_DIR = sonic-slave-jessie
 endif
-SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}')
+SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}')
 SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
 SLAVE_BASE_IMAGE = $(SLAVE_DIR)
 SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
@@ -109,6 +109,10 @@ DOCKER_RUN := docker run --rm=true --privileged \
 
 include rules/config
 
+ifneq ($(DEFAULT_CONTAINER_REGISTRY),)
+override DEFAULT_CONTAINER_REGISTRY := $(DEFAULT_CONTAINER_REGISTRY)/
+endif
+
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
 endif
@@ -185,6 +189,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            EXTRA_JESSIE_TARGETS=$(EXTRA_JESSIE_TARGETS) \
+                           DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

azure-pipelines.yml

@@ -30,6 +30,14 @@ resources:
     type: github
     name: Azure/sonic-mgmt
     endpoint: build
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    endpoint: build
+    ref: master
+
+variables:
+- template: .azure-pipelines/template-variables.yml@buildimage
 
 stages:
 - stage: Build

dockers/docker-base-stretch/Dockerfile.j2

@@ -1,10 +1,11 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
 {% if CONFIGURED_ARCH == "armhf" %}
-FROM multiarch/debian-debootstrap:armhf-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-stretch
 {% elif CONFIGURED_ARCH == "arm64" %}
-FROM multiarch/debian-debootstrap:arm64-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
 {% else %}
-FROM debian:stretch
+FROM {{ prefix }}debian:stretch
 {% endif %}
 
 # Clean documentation in FROM image

dockers/docker-base/Dockerfile.j2

@@ -1,9 +1,10 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% if CONFIGURED_ARCH == "armhf" %}
-FROM multiarch/debian-debootstrap:armhf-jessie
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-jessie
 {% elif CONFIGURED_ARCH == "arm64" %}
-FROM multiarch/debian-debootstrap:arm64-jessie
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-jessie
 {% else %}
-FROM debian:jessie
+FROM {{ prefix }}/debian:jessie
 {% endif %}
 
 ## Remove retired jessie-updates repo

dockers/docker-ptf/Dockerfile.j2

@@ -1,9 +1,10 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% if CONFIGURED_ARCH == "armhf" %}
-FROM multiarch/debian-debootstrap:armhf-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-stretch
 {% elif CONFIGURED_ARCH == "arm64" %}
-FROM multiarch/debian-debootstrap:arm64-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
 {% else %}
-FROM debian:stretch
+FROM {{ prefix }}debian:stretch
 {% endif %}
 
 MAINTAINER Pavel Shirshov

dockers/docker-sonic-mgmt/Dockerfile.j2

@@ -1,4 +1,5 @@
-FROM ubuntu:16.04
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
+FROM {{ prefix }}ubuntu:16.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 

rules/config

@@ -125,3 +125,6 @@ KUBERNETES_VERSION = 1.18.6
 KUBERNETES_CNI_VERSION = 0.8.6
 K8s_GCR_IO_PAUSE_VERSION = 3.2
 
+# reset default container registry from dockerhub to other
+DEFAULT_CONTAINER_REGISTRY ?=
+

slave.mk

@@ -85,6 +85,7 @@ list :
 ###############################################################################
 
 include $(RULES_PATH)/config
+export DEFAULT_CONTAINER_REGISTRY
 
 ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y)
 ENABLE_PFCWD_ON_START = y
@@ -213,6 +214,7 @@ $(info "INCLUDE_RESTAPI"                 : "$(INCLUDE_RESTAPI)")
 $(info "INCLUDE_SFLOW"                   : "$(INCLUDE_SFLOW)")
 $(info "INCLUDE_NAT"                     : "$(INCLUDE_NAT)")
 $(info "INCLUDE_KUBERNETES"              : "$(INCLUDE_KUBERNETES)")
+$(info "DEFAULT_CONTAINER_REGISTRY"      : "$(DEFAULT_CONTAINER_REGISTRY)")
 $(info )
 
 ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y)

sonic-slave-jessie/Dockerfile.j2

@@ -1,9 +1,10 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {%- if CONFIGURED_ARCH == "armhf"  -%}
-FROM multiarch/debian-debootstrap:armhf-jessie
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-jessie
 {%- elif CONFIGURED_ARCH == "arm64" -%}
-FROM multiarch/debian-debootstrap:arm64-jessie
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-jessie
 {%- else -%}
-FROM debian:jessie
+FROM {{ prefix }}debian:jessie
 {%- endif %}
 
 MAINTAINER johnar@microsoft.com

sonic-slave-stretch/Dockerfile.j2

@@ -1,9 +1,10 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {%- if CONFIGURED_ARCH == "armhf" %}
-FROM multiarch/debian-debootstrap:armhf-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-stretch
 {%- elif CONFIGURED_ARCH == "arm64" %}
-FROM multiarch/debian-debootstrap:arm64-stretch
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
 {%- else -%}
-FROM debian:stretch
+FROM {{ prefix }}debian:stretch
 {%- endif %}
 
 MAINTAINER gulv@microsoft.com