[ci]: add native arm64/armhf sonic-slave build pipeline (#9056)

setup sonic-slave template and use the template for amd64, arm64 and armhf sonic-slave Signed-off-by: Guohan Lu <lguohan@gmail.com>
2021-11-06 03:42:51 -07:00 · 2021-11-06 03:42:51 -07:00 · 7cfce6f3f2
commit 7cfce6f3f2
parent 3fc42df0bc
4 changed files with 201 additions and 61 deletions
--- a/.azure-pipelines/docker-sonic-slave-arm64.yml
+++ b/.azure-pipelines/docker-sonic-slave-arm64.yml
@ -0,0 +1,44 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - sonic-slave-jessie
+    - sonic-slave-stretch
+    - sonic-slave-buster
+    - sonic-slave-bullseye
+
+parameters:
+- name: 'dists'
+  type: object
+  default:
+  - bullseye
+  - buster
+  - stretch
+
+stages:
+- stage: Build
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
+      - template: docker-sonic-slave-template.yml
+        parameters:
+          pool: sonicbld-arm64
+          arch: arm64
+          dist: ${{ dist }}
--- a/.azure-pipelines/docker-sonic-slave-armhf.yml
+++ b/.azure-pipelines/docker-sonic-slave-armhf.yml
@ -0,0 +1,44 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - sonic-slave-jessie
+    - sonic-slave-stretch
+    - sonic-slave-buster
+    - sonic-slave-bullseye
+
+parameters:
+- name: 'dists'
+  type: object
+  default:
+  - bullseye
+  - buster
+  - stretch
+
+stages:
+- stage: Build
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
+      - template: docker-sonic-slave-template.yml
+        parameters:
+          pool: sonicbld-armhf
+          arch: armhf
+          dist: ${{ dist }}
--- a/.azure-pipelines/docker-sonic-slave-template.yml
+++ b/.azure-pipelines/docker-sonic-slave-template.yml
@ -0,0 +1,108 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+
+parameters:
+- name: arch
+  type: string
+  values:
+  - amd64
+  - armhf
+  - arm64
+- name: dist
+  type: string
+  values:
+  - bullseye
+  - buster
+  - stretch
+  - jessie
+- name: registry_url
+  type: string
+  default: sonicdev-microsoft.azurecr.io
+- name: registry_conn
+  type: string
+  default: sonicdev
+- name: pool
+  type: string
+  default: sonicbld
+  values:
+  - sonicbld
+  - sonicbld-arm64
+  - sonicbld-armhf
+
+jobs:
+- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
+  timeoutInMinutes: 360
+  pool: ${{ parameters.pool }}
+  steps:
+  - template: cleanup.yml
+  - checkout: self
+    clean: true
+    submodules: recursive
+  - bash: |
+      set -ex
+
+      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
+      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
+        if [ x${{ parameters.arch }} == x"amd64" ]; then
+          SLAVE_BASE_IMAGE=${SLAVE_DIR}
+          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
+        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
+          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
+          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
+        fi
+      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
+        SLAVE_BASE_IMAGE=${SLAVE_DIR}
+        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
+      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
+        SLAVE_BASE_IMAGE=${SLAVE_DIR}
+        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
+      else
+        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
+        exit 1
+      fi
+
+      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
+        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+      fi
+
+      containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
+      if [ ! -z "$containers" ]; then
+        docker container kill $containers || true
+        sleep 5
+      fi
+      images=$(docker images 'sonic-slave-*' -a -q)
+      if [ ! -z "$images" ]; then
+        docker rmi -f $images
+      fi
+
+      tmpfile=$(mktemp)
+
+      echo ${{ parameters.arch }} > .arch
+
+      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
+      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
+      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
+
+      mkdir -p target
+
+      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
+      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
+      set +x
+      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
+      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
+    env:
+      REGISTRY_SERVER: ${{ parameters.registry_url }}
+    displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
+
+  - task: Docker@2
+    displayName: Upload image
+    inputs:
+      containerRegistry: ${{ parameters.registry_conn }}
+      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
+      command: push
+      tags: |
+        $(VARIABLE_SLAVE_BASE_TAG)
+        latest
--- a/.azure-pipelines/docker-sonic-slave.yml
+++ b/.azure-pipelines/docker-sonic-slave.yml
@ -45,70 +45,14 @@ parameters:
  type: string
  default: sonicdev

-variables:
- ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
-  - name: BUILD_OPTIONS
-    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
-
 stages:
 - stage: Build
  jobs:
  - ${{ each dist in parameters.dists }}:
    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
      - ${{ each arch in parameters.arches }}:
-        - job: Build_${{ dist }}_${{ arch }}
-          timeoutInMinutes: 360
-          pool: sonicbld
-          steps:
-          - template: cleanup.yml
-          - checkout: self
-            clean: true
-            submodules: recursive
-          - bash: |
-              set -ex
-
-              containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-              if [ ! -z "$containers" ]; then
-                docker container kill $containers || true
-                sleep 5
-              fi
-              images=$(docker images 'sonic-slave-*' -a -q)
-              if [ ! -z "$images" ]; then
-                docker rmi -f $images
-              fi
-
-              SLAVE_DIR=sonic-slave-${{ dist }}
-              if [ x${{ arch }} == x"amd64" ]; then
-                SLAVE_BASE_IMAGE=${SLAVE_DIR}
-              else
-                SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ arch }}
-              fi
-
-              tmpfile=$(mktemp)
-
-              echo ${{ arch }} > .arch
-
-              DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-              SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-              SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-              mkdir -p target
-
-              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:latest
-              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG
-              set +x
-              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE"
-              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
-            env:
-              REGISTRY_SERVER: ${{ parameters.registry_url }}
-            displayName: Build sonic-slave-${{ dist }}-${{ arch }}
-
-          - task: Docker@2
-            displayName: Upload image
-            inputs:
-              containerRegistry: ${{ parameters.registry_conn }}
-              repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-              command: push
-              tags: |
-                $(VARIABLE_SLAVE_BASE_TAG)
-                latest
+        - template: docker-sonic-slave-template.yml
+          parameters:
+            pool: sonicbld
+            arch: ${{ arch }}
+            dist: ${{ dist }}