From 8211c850f163c0e25047b550b1e8199f9b3a1e48 Mon Sep 17 00:00:00 2001
From: Liu Shilong <shilongliu@microsoft.com>
Date: Wed, 21 Sep 2022 15:50:30 +0800
Subject: [PATCH] [ci] Update docker sonic slave pipeline to build slave base
 docker (#11908)

* [ci] Update docker sonic slave pipeline to build slave base docker
---
 .azure-pipelines/docker-sonic-slave-arm64.yml | 52 ----------
 .azure-pipelines/docker-sonic-slave-armhf.yml | 52 ----------
 .../docker-sonic-slave-template.yml           | 94 +++++--------------
 .azure-pipelines/docker-sonic-slave.yml       | 44 +++++----
 4 files changed, 52 insertions(+), 190 deletions(-)
 delete mode 100644 .azure-pipelines/docker-sonic-slave-arm64.yml
 delete mode 100644 .azure-pipelines/docker-sonic-slave-armhf.yml

diff --git a/.azure-pipelines/docker-sonic-slave-arm64.yml b/.azure-pipelines/docker-sonic-slave-arm64.yml
deleted file mode 100644
index 6350d8fd3b..0000000000
--- a/.azure-pipelines/docker-sonic-slave-arm64.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
-# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
-resources:
-  repositories:
-  - repository: buildimage
-    type: github
-    name: sonic-net/sonic-buildimage
-    ref: master
-    endpoint: sonic-net
-
-schedules:
-- cron: "0 8 * * *"
-  branches:
-    include:
-    - master
-    - 202012
-  always: true
-
-trigger: none
-pr:
-  branches:
-    include:
-    - master
-  paths:
-    include:
-    - sonic-slave-jessie
-    - sonic-slave-stretch
-    - sonic-slave-buster
-    - sonic-slave-bullseye
-    - .azure-pipelines
-
-parameters:
-- name: 'dists'
-  type: object
-  default:
-  - bullseye
-  - buster
-  - stretch
-
-stages:
-- stage: Build
-  jobs:
-  - ${{ each dist in parameters.dists }}:
-    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
-      - template: docker-sonic-slave-template.yml
-        parameters:
-          pool: sonicbld-arm64
-          arch: arm64
-          dist: ${{ dist }}
diff --git a/.azure-pipelines/docker-sonic-slave-armhf.yml b/.azure-pipelines/docker-sonic-slave-armhf.yml
deleted file mode 100644
index fcdad5f2cd..0000000000
--- a/.azure-pipelines/docker-sonic-slave-armhf.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
-# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
-resources:
-  repositories:
-  - repository: buildimage
-    type: github
-    name: sonic-net/sonic-buildimage
-    ref: master
-    endpoint: sonic-net
-
-schedules:
-- cron: "0 8 * * *"
-  branches:
-    include:
-    - master
-    - 202012
-  always: true
-
-trigger: none
-pr:
-  branches:
-    include:
-    - master
-  paths:
-    include:
-    - sonic-slave-jessie
-    - sonic-slave-stretch
-    - sonic-slave-buster
-    - sonic-slave-bullseye
-    - .azure-pipelines
-
-parameters:
-- name: 'dists'
-  type: object
-  default:
-  - bullseye
-  - buster
-  - stretch
-
-stages:
-- stage: Build
-  jobs:
-  - ${{ each dist in parameters.dists }}:
-    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
-      - template: docker-sonic-slave-template.yml
-        parameters:
-          pool: sonicbld-armhf
-          arch: armhf
-          dist: ${{ dist }}
diff --git a/.azure-pipelines/docker-sonic-slave-template.yml b/.azure-pipelines/docker-sonic-slave-template.yml
index 2cf06638a4..1a7b983b5c 100644
--- a/.azure-pipelines/docker-sonic-slave-template.yml
+++ b/.azure-pipelines/docker-sonic-slave-template.yml
@@ -10,6 +10,9 @@ parameters:
   - amd64
   - armhf
   - arm64
+- name: march
+  type: string
+  default: ''
 - name: dist
   type: string
   values:
@@ -32,89 +35,44 @@ parameters:
   - sonicbld-armhf
 
 jobs:
-- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
+- job: Build_${{ parameters.dist }}_${{ parameters.march }}${{ parameters.arch }}
   timeoutInMinutes: 360
+  variables:
+  - template: .azure-pipelines/template-variables.yml@buildimage
+  - template: .azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
   pool: ${{ parameters.pool }}
   steps:
   - template: cleanup.yml
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-    - template: template-clean-sonic-slave.yml
-  - ${{ else }}:
-    - template: '/.azure-pipelines/template-clean-sonic-slave.yml@buildimage'
+  - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
   - checkout: self
     clean: true
     submodules: recursive
+  - task: Docker@2
+    displayName: Login to ACR
+    inputs:
+      command: login
+      containerRegistry: ${{ parameters.registry_conn }}
   - bash: |
       set -ex
+      image_tag=$(BLDENV=${{ parameters.dist }} make -f Makefile.work showtag PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} | grep sonic-slave | tail -n 1)
+      image_latest=$(echo $(echo $image_tag | awk -F: '{print$1}'):latest)
+      docker rmi $image_tag || true
 
-      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
-      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
-        if [ x${{ parameters.arch }} == x"amd64" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
-        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
-        fi
-      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
-      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
-      else
-        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
-        exit 1
+      if [[ "$(Build.Reason)" =~ [a-zA-Z]*CI ]] && docker pull ${{ parameters.registry_url }}/${image_tag};then
+        exit 0
       fi
 
-      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
-        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker make configure PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} $args || docker image ls $image_tag
+      if [[ "$(Build.Reason)" == "PullRequest" ]];then
+        exit 0
       fi
 
-      tmpfile=$(mktemp)
-
-      echo ${{ parameters.arch }} > .arch
-
-      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-      mkdir -p target
-
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
-      if [ "$SLAVE_BASE_IMAGE_UPLOAD" != "$SLAVE_DIR" ]; then
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:latest-${{ parameters.arch }}
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:$SLAVE_BASE_TAG
+      docker tag ${image_tag} ${REGISTRY_SERVER}/${image_tag}
+      docker push ${REGISTRY_SERVER}/${image_tag}
+      if [[ "${{ parameters.arch }}" == "amd64" ]];then
+        docker tag ${image_tag} ${REGISTRY_SERVER}/${image_latest}
+        docker push ${REGISTRY_SERVER}/${image_latest}
       fi
-      set +x
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
     env:
       REGISTRY_SERVER: ${{ parameters.registry_url }}
     displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
-
-  - task: Docker@2
-    condition: ne(variables['Build.Reason'], 'PullRequest')
-    displayName: Upload image
-    inputs:
-      containerRegistry: ${{ parameters.registry_conn }}
-      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-      command: push
-      ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-          latest
-      ${{ else }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-  - ${{ if ne(parameters.arch, 'amd64') }}:
-    - task: Docker@2
-      condition: ne(variables['Build.Reason'], 'PullRequest')
-      displayName: Upload image ${{ parameters.dist }}
-      inputs:
-        containerRegistry: ${{ parameters.registry_conn }}
-        repository: "sonic-slave-${{ parameters.dist }}"
-        command: push
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG) 
diff --git a/.azure-pipelines/docker-sonic-slave.yml b/.azure-pipelines/docker-sonic-slave.yml
index cd8a6df4af..dd7a11d2c5 100644
--- a/.azure-pipelines/docker-sonic-slave.yml
+++ b/.azure-pipelines/docker-sonic-slave.yml
@@ -12,26 +12,28 @@ resources:
     endpoint: sonic-net
 
 schedules:
-- cron: "0 8 * * *"
+- cron: "0 0 * * 0"
+  displayName: Weekly build
   branches:
     include:
     - master
-    - 202012
+    - 202???
   always: true
 
-trigger: none
-pr:
+pr: none
+trigger:
+  batch: true
   branches:
     include:
     - master
+    - 202???
   paths:
     include:
-    - sonic-slave-jessie
-    - sonic-slave-stretch
-    - sonic-slave-buster
-    - sonic-slave-bullseye
+    - sonic-slave-*
     - src/sonic-build-hooks
-    - .azure-pipelines
+    - files/build/versions
+    - Makefile
+    - Makefile.work
 
 parameters:
 - name: 'arches'
@@ -60,15 +62,21 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-          - template: docker-sonic-slave-template.yml
+        - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
+          parameters:
+            pool: sonicbld
+            arch: ${{ arch }}
+            dist: ${{ dist }}
+- stage: Build_march
+  dependsOn: []
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
+      - ${{ each arch in parameters.arches }}:
+        - ${{ if ne(arch, 'amd64') }}:
+          - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
             parameters:
-              pool: sonicbld
-              arch: ${{ arch }}
-              dist: ${{ dist }}
-        - ${{ else }}:
-          - template: '/.azure-pipelines/docker-sonic-slave-template.yml@buildimage'
-            parameters:
-              pool: sonicbld
+              pool: sonicbld-${{ arch }}
               arch: ${{ arch }}
               dist: ${{ dist }}
+              march: march_