From 78737775d9e84d306799edf2b56c4768b26a25af Mon Sep 17 00:00:00 2001
From: Guohan Lu <lguohan@gmail.com>
Date: Fri, 12 Mar 2021 00:44:09 -0800
Subject: [PATCH] [ci]: add official-build for 202012

Signed-off-by: Guohan Lu <lguohan@gmail.com>
---
 .azure-pipelines/build-template.yml    | 116 +++++++++++++++++++++++++
 .azure-pipelines/cleanup.yml           |  17 ++++
 .azure-pipelines/official-build.yml    |  62 +++++++++++++
 .azure-pipelines/run-test-template.yml |  82 +++++++++++++++++
 4 files changed, 277 insertions(+)
 create mode 100644 .azure-pipelines/build-template.yml
 create mode 100644 .azure-pipelines/cleanup.yml
 create mode 100644 .azure-pipelines/official-build.yml
 create mode 100644 .azure-pipelines/run-test-template.yml

diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
new file mode 100644
index 0000000000..c1cc04836a
--- /dev/null
+++ b/.azure-pipelines/build-template.yml
@@ -0,0 +1,116 @@
+parameters:
+- name: platform
+  type: string
+  values:
+  - broadcom
+  - centec-arm64
+  - marvell-armhf
+  - mellanox
+  - vs
+
+- name: platform_arch
+  type: string
+  values:
+  - amd64
+  - armhf
+  - arm64
+  default: amd64
+
+- name: platform_short
+  type: string
+  values:
+  - brcm
+  - centec-arm64
+  - marvell-armhf
+  - mlnx
+  - vs
+
+- name: cache_mode
+  type: string
+  values:
+  - wcache
+  - rcache
+  - cache
+
+- name: pool
+  type: string
+  values:
+  - sonicbld
+  - sonicbld_8c
+  default: sonicbld
+
+- name: dbg_image
+  type: boolean
+  default: false
+
+- name: swi_image
+  type: boolean
+  default: false
+
+- name: raw_image
+  type: boolean
+  default: false
+
+- name: sync_rpc_image
+  type: boolean
+  default: false
+
+- name: timeout
+  type: number
+  default: 600
+
+jobs:
+- job:
+  pool: ${{ parameters.pool }}
+  displayName: ${{ parameters.platform }}
+  timeoutInMinutes: ${{ parameters.timeout }}
+  steps:
+  - template: cleanup.yml
+  - checkout: self
+    clean: true
+    submodules: recursive
+    displayName: 'Checkout code'
+  - script: |
+      git submodule foreach --recursive 'git clean -xfdf || true'
+      git submodule foreach --recursive 'git reset --hard || true'
+      git submodule foreach --recursive 'git remote update || true'
+      git submodule update --init --recursive
+    displayName: 'Reset submodules'
+  - script: |
+      set -e
+      sudo modprobe overlay
+      sudo apt-get install -y acl
+      export DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker
+      CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/${{ parameters.platform }}"
+      ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=${{ parameters.platform }} PLATFORM_ARCH=${{ parameters.platform_arch }}
+      trap "sudo rm -rf fsroot" EXIT
+
+      if [ ${{ parameters.platform }} == vs ]; then
+        if [ ${{ parameters.dbg_image }} == true ]; then
+          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && \
+            mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+        fi
+
+        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+      else
+        if [ ${{ parameters.dbg_image }} == true ]; then
+          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \
+            mv target/sonic-${{ parameters.platform }}.bin target/sonic-${{ parameters.platform }}-dbg.bin
+        fi
+        if [ ${{ parameters.swi_image }} == true ]; then
+          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
+        fi
+        if [ ${{ parameters.raw_image }} == true ]; then
+          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
+        fi
+        if [ ${{ parameters.sync_rpc_image }} == true ]; then
+          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+        fi
+
+        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin
+      fi
+    displayName: 'Build sonic image'
+  - template: cleanup.yml
+  - publish: $(System.DefaultWorkingDirectory)/
+    artifact: sonic-buildimage.${{ parameters.platform }}
+    displayName: "Archive sonic image"
diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml
new file mode 100644
index 0000000000..ffbcf298b6
--- /dev/null
+++ b/.azure-pipelines/cleanup.yml
@@ -0,0 +1,17 @@
+steps:
+- script: |
+    if sudo [ -f /var/run/march/docker.pid ] ; then
+        pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
+    fi
+    sudo rm -f /var/run/march/docker.pid
+    sudo rm -rf /data/march/docker
+
+    # clean native docker build
+    if sudo [ -f dockerfs/var/run/docker.pid ] ; then
+        pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
+    fi
+    sudo rm -rf dockerfs
+    sudo rm -rf fsroot
+    username=$(id -un)
+    sudo chown -R ${username}.${username} .
+  displayName: "Clean Workspace"
diff --git a/.azure-pipelines/official-build.yml b/.azure-pipelines/official-build.yml
new file mode 100644
index 0000000000..8524027d7b
--- /dev/null
+++ b/.azure-pipelines/official-build.yml
@@ -0,0 +1,62 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+schedules:
+- cron: "0 8 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+
+  jobs:
+  - template: build-template.yml
+    parameters:
+      platform: broadcom
+      platform_short: brcm
+      cache_mode: wcache
+      dbg_image: true
+      swi_image: true
+      raw_image: true
+      sync_rpc_image: true
+
+  - template: build-template.yml
+    parameters:
+      platform: mellanox
+      platform_short: mlnx
+      cache_mode: wcache
+      dbg_image: true
+      sync_rpc_image: true
+
+  - template: build-template.yml
+    parameters:
+      platform: vs
+      platform_short: vs
+      dbg_image: true
+      cache_mode: wcache
+
+  - template: build-template.yml
+    parameters:
+      timeout: 3600
+      platform: marvell-armhf
+      platform_arch: armhf
+      platform_short: marvell-armhf
+      cache_mode: wcache
+      pool: sonicbld_8c
+
+  - template: build-template.yml
+    parameters:
+      timeout: 3600
+      platform: centec-arm64
+      platform_arch: arm64
+      platform_short: centec-arm64
+      cache_mode: wcache
+      pool: sonicbld_8c
diff --git a/.azure-pipelines/run-test-template.yml b/.azure-pipelines/run-test-template.yml
new file mode 100644
index 0000000000..afe5eadec3
--- /dev/null
+++ b/.azure-pipelines/run-test-template.yml
@@ -0,0 +1,82 @@
+parameters:
+- name: dut
+  type: string
+- name: tbname
+  type: string
+- name: tbtype
+  type: string
+- name: ptf_name
+  type: string
+
+steps:
+- checkout: self
+  clean: true
+  displayName: 'checkout sonic-mgmt repo'
+
+- task: DownloadPipelineArtifact@2
+  inputs:
+    artifact: sonic-buildimage.vs
+  displayName: "Download sonic-buildimage.vs artifact"
+
+- script: |
+    set -x
+    sudo mkdir -p /data/sonic-vm/images
+    sudo cp -v ../target/sonic-vs.img.gz /data/sonic-vm/images/sonic-vs.img.gz
+    sudo gzip -fd /data/sonic-vm/images/sonic-vs.img.gz
+    username=$(id -un)
+    sudo chown -R $username.$username /data/sonic-vm
+
+    pushd /data/sonic-mgmt
+    git remote update
+    git reset --hard origin/master
+    sed -i s/use_own_value/${username}/ ansible/veos_vtb
+    echo aaa > ansible/password.txt
+    docker exec sonic-mgmt bash -c "pushd /data/sonic-mgmt/ansible;./testbed-cli.sh -d /data/sonic-vm -m $(inventory) -t $(testbed_file) -k ceos refresh-dut ${{ parameters.tbname }} password.txt" && sleep 180
+  displayName: "Setup testbed"
+
+- script: |
+    rm -rf $(Build.ArtifactStagingDirectory)/*
+    docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }}"
+  displayName: "Run tests"
+
+- script: |
+    # save dut state if test fails
+    virsh_version=$(virsh --version)
+    if [ $virsh_version == "6.0.0" ]; then
+        mkdir -p $(Build.ArtifactStagingDirectory)/kvmdump
+        virsh -c qemu:///system list
+        virsh -c qemu:///system save ${{ parameters.dut }} $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.memdmp
+        virsh -c qemu:///system dumpxml ${{ parameters.dut }} > $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.xml
+        img=$(virsh -c qemu:///system domblklist ${{ parameters.dut }} | grep vda | awk '{print $2}')
+        cp $img $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.img
+        virsh -c qemu:///system undefine ${{ parameters.dut }}
+    fi
+
+    docker commit ${{ parameters.ptf_name }} docker-ptf:$(Build.BuildNumber)
+    docker save docker-ptf:$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/kvmdump/docker-ptf-dump.gz
+    docker rmi docker-ptf:$(Build.BuildNumber)
+  displayName: "Collect kvmdump"
+  condition: failed()
+
+- script: |
+    cp -r /data/sonic-mgmt/tests/logs $(Build.ArtifactStagingDirectory)/
+    username=$(id -un)
+    sudo chown -R $username.$username $(Build.ArtifactStagingDirectory)
+  displayName: "Collect test logs"
+  condition: succeededOrFailed()
+
+- publish: $(Build.ArtifactStagingDirectory)/kvmdump
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype}}.memdump@$(System.JobAttempt)
+  displayName: "Archive sonic kvm memdump"
+  condition: failed()
+
+- publish: $(Build.ArtifactStagingDirectory)/logs
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype }}.log@$(System.JobAttempt)
+  displayName: "Archive sonic kvm logs"
+  condition: succeededOrFailed()
+
+- task: PublishTestResults@2
+  inputs:
+    testResultsFiles: '$(Build.ArtifactStagingDirectory)/logs/**/*.xml'
+    testRunTitle: kvmtest.${{ parameters.tbtype }}
+  condition: succeededOrFailed()