[ci] Update docker sonic slave pipeline to build slave base docker (#11908) (#12405)

* [ci] Update docker sonic slave pipeline to build slave base docker

.azure-pipelines/docker-sonic-slave-template.yml

@@ -1,106 +0,0 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
-# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
-resources:
-  repositories:
-  - repository: buildimage
-    type: github
-    name: sonic-net/sonic-buildimage
-    ref: master
-    endpoint: sonic-net
-
-parameters:
-- name: arch
-  type: string
-  values:
-  - amd64
-  - armhf
-  - arm64
-- name: dist
-  type: string
-  values:
-  - bullseye
-  - buster
-  - stretch
-  - jessie
-- name: registry_url
-  type: string
-  default: sonicdev-microsoft.azurecr.io
-- name: registry_conn
-  type: string
-  default: sonicdev
-- name: pool
-  type: string
-  default: sonicbld
-  values:
-  - sonicbld
-  - sonicbld-arm64
-  - sonicbld-armhf
-
-jobs:
-- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
-  timeoutInMinutes: 360
-  pool: ${{ parameters.pool }}
-  steps:
-  - template: cleanup.yml
-  - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
-  - checkout: self
-    clean: true
-    submodules: recursive
-  - bash: |
-      set -ex
-
-      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
-      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
-        if [ x${{ parameters.arch }} == x"amd64" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
-        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
-        fi
-      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
-      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
-      else
-        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
-        exit 1
-      fi
-
-      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
-        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
-      fi
-
-      tmpfile=$(mktemp)
-
-      echo ${{ parameters.arch }} > .arch
-
-      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-      mkdir -p target
-
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
-      set +x
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
-    env:
-      REGISTRY_SERVER: ${{ parameters.registry_url }}
-    displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
-
-  - task: Docker@2
-    displayName: Upload image
-    inputs:
-      containerRegistry: ${{ parameters.registry_conn }}
-      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-      command: push
-      tags: |
-        $(VARIABLE_SLAVE_BASE_TAG)
-        latest

.azure-pipelines/docker-sonic-slave.yml

@@ -3,25 +3,37 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: sonic-net/sonic-buildimage
+    ref: master
+    endpoint: sonic-net
 
 schedules:
-- cron: "0 8 * * *"
+- cron: "0 0 * * 0"
-  branches:
+  displayName: Weekly build
-    include:
-    - 202012
-  always: true
-
-trigger: none
-pr:
   branches:
     include:
     - master
+    - 202???
+  always: true
+
+pr: none
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+    - 202???
   paths:
     include:
-    - sonic-slave-jessie
+    - sonic-slave-*
-    - sonic-slave-stretch
-    - sonic-slave-buster
     - src/sonic-build-hooks
+    - files/build/versions
+    - Makefile
+    - Makefile.work
 
 parameters:
 - name: 'arches'
@@ -43,70 +55,27 @@ parameters:
   type: string
   default: sonicdev
 
-variables:
-- ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
-  - name: BUILD_OPTIONS
-    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
-
 stages:
 - stage: Build
   jobs:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - job: Build_${{ dist }}_${{ arch }}
+        - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
-          timeoutInMinutes: 360
+          parameters:
             pool: sonicbld
-          steps:
+            arch: ${{ arch }}
-          - template: cleanup.yml
+            dist: ${{ dist }}
-          - checkout: self
+- stage: Build_march
-            clean: true
+  dependsOn: []
-            submodules: recursive
+  jobs:
-          - bash: |
+  - ${{ each dist in parameters.dists }}:
-              set -ex
+    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
-
+      - ${{ each arch in parameters.arches }}:
-              containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
+        - ${{ if ne(arch, 'amd64') }}:
-              if [ ! -z "$containers" ]; then
+          - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
-                docker container kill $containers || true
+            parameters:
-                sleep 5
+              pool: sonicbld-${{ arch }}
-              fi
+              arch: ${{ arch }}
-              images=$(docker images 'sonic-slave-*' -a -q)
+              dist: ${{ dist }}
-              if [ ! -z "$images" ]; then
+              march: march_
-                docker rmi -f $images
-              fi
-
-              SLAVE_DIR=sonic-slave-${{ dist }}
-              if [ x${{ arch }} == x"amd64" ]; then
-                SLAVE_BASE_IMAGE=${SLAVE_DIR}
-              else
-                SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ arch }}
-              fi
-
-              tmpfile=$(mktemp)
-
-              echo ${{ arch }} > .arch
-
-              DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-              SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-              SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-              mkdir -p target
-
-              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:latest
-              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG
-              set +x
-              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE"
-              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
-            env:
-              REGISTRY_SERVER: ${{ parameters.registry_url }}
-            displayName: Build sonic-slave-${{ dist }}-${{ arch }}
-
-          - task: Docker@2
-            displayName: Upload image
-            inputs:
-              containerRegistry: ${{ parameters.registry_conn }}
-              repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-              command: push
-              tags: |
-                $(VARIABLE_SLAVE_BASE_TAG)
-                latest