[CI] Add azure pipeline file for official build

Signed-off-by: Shilong Liu <shilongliu@microsoft.com>
This commit is contained in:
Shilong Liu 2021-04-22 10:11:11 +08:00 committed by lguohan
parent c91b02cb1b
commit bf02ba16ee
8 changed files with 183 additions and 102 deletions

View File

@ -18,10 +18,25 @@ pool: sonicbld
stages: stages:
- stage: Build - stage: Build
variables:
CACHE_MODE: none
VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS='
jobs: jobs:
- template: azure-pipelines-build.yml - template: azure-pipelines-build.yml
parameters: parameters:
buildSlave: y buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
preSteps:
- script: |
containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
if [ ! -z "$containers" ]; then
docker container kill $containers || true
sleep 5
fi
images=$(docker images 'sonic-slave-*' -a -q)
if [ ! -z "$images" ]; then
docker rmi -f $images
fi
displayName: 'Cleanup sonic slave'
- stage: UpgradeVersions - stage: UpgradeVersions
jobs: jobs:
- job: UpgradeVersions - job: UpgradeVersions
@ -48,11 +63,14 @@ stages:
cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/ cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
make freeze FREEZE_VERSION_OPTIONS=-r make freeze FREEZE_VERSION_OPTIONS=-r
find files/build/versions find files/build/versions
for artifact in $artifacts ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
for artifact in $ordered_artifacts
do do
rm -rf target/versions rm -rf target/versions
cp -r $artifact/versions target/ cp -r $artifact/versions target/
make freeze FREEZE_VERSION_OPTIONS="-a -d" OPTIONS="-a -d"
[[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"
done done
git diff files/build/versions git diff files/build/versions
displayName: 'Freeze Versions' displayName: 'Freeze Versions'

View File

@ -1,3 +1,5 @@
# The azure pipeline template for Official build, and upgrade version build
parameters: parameters:
- name: 'jobFilters' - name: 'jobFilters'
type: object type: object
@ -5,68 +7,95 @@ parameters:
- name: 'buildOptions' - name: 'buildOptions'
type: string type: string
default: 'SONIC_CONFIG_BUILD_JOBS=1' default: 'SONIC_CONFIG_BUILD_JOBS=1'
- name: 'buildSlave' - name: 'preSteps'
type: string type: stepList
default: 'n' default: []
- name: 'postSteps' - name: 'postSteps'
type: stepList type: stepList
default: [] default: []
jobs: jobs:
- template: azure-pipelines-job-groups.yml - template: azure-pipelines-image-template.yml
parameters: parameters:
jobFilters: ${{ parameters.jobFilters }} jobFilters: ${{ parameters.jobFilters }}
preSteps: preSteps: ${{ parameters.preSteps }}
- script: | postSteps: ${{ parameters.postSteps }}
containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }') jobVariables:
if [ ! -z "$containers" ]; then PLATFORM_AZP: $(GROUP_NAME)
docker container kill $containers || true PLATFORM_ARCH: amd64
sleep 5 BUILD_OPTIONS: ${{ parameters.buildOptions }}
fi DOCKER_DATA_ROOT_FOR_MULTIARCH: /data/march/docker
if [ "${{ parameters.buildSlave }}" == "y" ]; then dbg_image: false
images=$(docker images 'sonic-slave-*' -a -q) swi_image: false
[ ! -z "$images" ] && docker rmi -f $images raw_image: false
fi sync_rpc_image: false
sudo rm -rf $(ls -A1) platform_rpc: false
displayName: 'Init'
- checkout: self
submodules: recursive
displayName: 'Checkout code'
- script: |
make ${{ parameters.buildOptions }} PLATFORM=$GROUP_NAME configure
displayName: 'Make configure'
postSteps:
- ${{ parameters.postSteps }}
- publish: $(System.DefaultWorkingDirectory)/target
artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
displayName: "Archive sonic image"
jobGroups: jobGroups:
- name: vs - name: vs
script: | variables:
sudo bash -c "echo 1 > /proc/sys/vm/compact_memory" dbg_image: true
make ${{ parameters.buildOptions }} target/sonic-vs.img.gz
- name: broadcom
script: |
make ${{ parameters.buildOptions }} target/sonic-broadcom.bin target/sonic-aboot-broadcom.swi
- name: barefoot - name: barefoot
script: | variables:
make ${{ parameters.buildOptions }} target/sonic-barefoot.bin target/sonic-aboot-barefoot.swi swi_image: true
- name: broadcom
variables:
dbg_image: true
swi_image: true
raw_image: true
sync_rpc_image: true
platform_rpc: brcm
- name: centec - name: centec
script: | variables:
make ${{ parameters.buildOptions }} INSTALL_DEBUG_TOOLS=y target/sonic-centec.bin dbg_image: true
mv target/sonic-centec.bin target/sonic-centec-dbg.bin sync_rpc_image: true
make ${{ parameters.buildOptions }} target/sonic-centec.bin platform_rpc: centec
make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y target/docker-syncd-centec-rpc.gz - name: centec-arm64
pool: sonicbld_8c
timeoutInMinutes: 1800
variables:
PLATFORM_ARCH: arm64
- name: generic
variables:
dbg_image: true
- name: innovium - name: innovium
script: | variables:
make ${{ parameters.buildOptions }} SONIC_CONFIG_BUILD_JOBS=1 target/sonic-innovium.bin swi_image: true
- name: marvell-armhf
pool: sonicbld_8c
timeoutInMinutes: 1800
variables:
PLATFORM_ARCH: armhf
- name: mellanox - name: mellanox
script: | variables:
make ${{ parameters.buildOptions }} target/sonic-mellanox.bin dbg_image: true
- name: mellanox sync_rpc_image: true
extName: _rpc platform_rpc: mlnx
script: |
make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y all
- name: nephos - name: nephos
script: | variables:
make ${{ parameters.buildOptions }} target/sonic-nephos.bin dbg_image: true
sync_rpc_image: true
platform_rpc: nephos
buildSteps:
- bash: |
if [ $(GROUP_NAME) == vs ]; then
if [ $(dbg_image) == true ]; then
make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
fi
make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
else
if [ $(dbg_image) == true ]; then
make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \
mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
fi
if [ $(swi_image) == true ]; then
make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
fi
if [ $(raw_image) == true ]; then
make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
fi
if [ $(sync_rpc_image) == true ]; then
make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
fi
make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
fi
displayName: "Build sonic image"

View File

@ -0,0 +1,56 @@
# The azure pipeline template for PR build, Official build, and upgrade version build
parameters:
- name: 'jobFilters'
type: object
default: ''
- name: 'preSteps'
type: stepList
default: []
- name: 'buildSteps'
type: stepList
default: []
- name: 'postSteps'
type: stepList
default: []
- name: jobGroups
type: object
default: []
- name: jobVariables
type: object
default: []
jobs:
- template: azure-pipelines-job-groups.yml
parameters:
jobFilters: ${{ parameters.jobFilters }}
jobVariables: ${{ parameters.jobVariables }}
preSteps:
- template: cleanup.yml
- ${{ parameters. preSteps }}
- script: |
if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
fi
displayName: "Set cache options"
- checkout: self
submodules: recursive
displayName: 'Checkout code'
- script: |
git checkout -b $(Build.SourceBranchName)
sudo modprobe overlay
sudo apt-get install -y acl
sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) configure
displayName: 'Make configure'
postSteps:
- script: cp target -r $(Build.ArtifactStagingDirectory)/
displayName: Copy Artifacts
- publish: $(Build.ArtifactStagingDirectory)
artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
displayName: "Archive sonic image"
- ${{ parameters.postSteps }}
- template: cleanup.yml
jobGroups: ${{ parameters.jobGroups }}
buildSteps: ${{ parameters.buildSteps }}

View File

@ -2,6 +2,9 @@ parameters:
- name: 'preSteps' - name: 'preSteps'
type: stepList type: stepList
default: [] default: []
- name: 'buildSteps'
type: stepList
default: []
- name: "postSteps" - name: "postSteps"
type: stepList type: stepList
default: [] default: []
@ -16,14 +19,14 @@ parameters:
default: '' default: ''
- name: 'timeoutInMinutes' - name: 'timeoutInMinutes'
type: 'number' type: 'number'
default: 1440 default: 600
- name: 'jobFilters' - name: 'jobFilters'
type: object type: object
default: '' default: ''
jobs: jobs:
- ${{ each jobGroup in parameters.jobGroups }}: - ${{ each jobGroup in parameters.jobGroups }}:
- ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name)) }}: - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name), endswith(variables['Build.DefinitionName'], format('.{0}{1}', jobGroup.name, jobGroup.extName))) }}:
- job: ${{ replace(format('{0}{1}', jobGroup.name, jobGroup.extName), '-', '_') }} - job: ${{ replace(format('{0}{1}', jobGroup.name, jobGroup.extName), '-', '_') }}
${{ each pair in jobGroup }}: ${{ each pair in jobGroup }}:
${{ if not(in(pair.key, 'job', 'name', 'extName', 'variables', 'steps', 'script', 'scriptEnv')) }}: ${{ if not(in(pair.key, 'job', 'name', 'extName', 'variables', 'steps', 'script', 'scriptEnv')) }}:
@ -50,4 +53,5 @@ jobs:
displayName: 'JobScript' displayName: 'JobScript'
- ${{ if ne(jobGroup.steps, '') }}: - ${{ if ne(jobGroup.steps, '') }}:
- ${{ jobGroup.steps }} - ${{ jobGroup.steps }}
- ${{ parameters.buildSteps }}
- ${{ parameters.postSteps }} - ${{ parameters.postSteps }}

View File

@ -114,3 +114,12 @@ jobs:
- publish: $(System.DefaultWorkingDirectory)/ - publish: $(System.DefaultWorkingDirectory)/
artifact: sonic-buildimage.${{ parameters.platform }} artifact: sonic-buildimage.${{ parameters.platform }}
displayName: "Archive sonic image" displayName: "Archive sonic image"
- script: |
set -x
find target -name "*.log" | xargs -I{} cp {} $(Build.ArtifactStagingDirectory)/
condition: failed()
displayName: "Copy build logs"
- publish: $(Build.ArtifactStagingDirectory)/
artifact: sonic-buildimage.${{ parameters.platform}}.logs@$(System.JobAttempt)
displayName: "Archive build logs"
condition: failed()

View File

@ -10,8 +10,5 @@ steps:
if sudo [ -f dockerfs/var/run/docker.pid ] ; then if sudo [ -f dockerfs/var/run/docker.pid ] ; then
pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
fi fi
sudo rm -rf dockerfs sudo rm -rf $(ls -A1)
sudo rm -rf fsroot
username=$(id -un)
sudo chown -R ${username}.${username} .
displayName: "Clean Workspace" displayName: "Clean Workspace"

View File

@ -17,47 +17,13 @@ pr: none
stages: stages:
- stage: Build - stage: Build
pool: sonicbld
variables:
CACHE_MODE: wcache
${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
jobs: jobs:
- template: build-template.yml - template: azure-pipelines-build.yml
parameters: parameters:
platform: broadcom buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
platform_short: brcm jobFilters: none
cache_mode: wcache
dbg_image: true
swi_image: true
raw_image: true
sync_rpc_image: true
- template: build-template.yml
parameters:
platform: mellanox
platform_short: mlnx
cache_mode: wcache
dbg_image: true
sync_rpc_image: true
- template: build-template.yml
parameters:
platform: vs
platform_short: vs
dbg_image: true
cache_mode: wcache
- template: build-template.yml
parameters:
timeout: 3600
platform: marvell-armhf
platform_arch: armhf
platform_short: marvell-armhf
cache_mode: wcache
pool: sonicbld_8c
- template: build-template.yml
parameters:
timeout: 3600
platform: centec-arm64
platform_arch: arm64
platform_short: centec-arm64
cache_mode: wcache
pool: sonicbld_8c

View File

@ -49,6 +49,8 @@ steps:
virsh -c qemu:///system dumpxml ${{ parameters.dut }} > $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.xml virsh -c qemu:///system dumpxml ${{ parameters.dut }} > $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.xml
img=$(virsh -c qemu:///system domblklist ${{ parameters.dut }} | grep vda | awk '{print $2}') img=$(virsh -c qemu:///system domblklist ${{ parameters.dut }} | grep vda | awk '{print $2}')
cp $img $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.img cp $img $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.img
gzip $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.img
sudo gzip $(Build.ArtifactStagingDirectory)/kvmdump/${{ parameters.dut }}.memdmp
virsh -c qemu:///system undefine ${{ parameters.dut }} virsh -c qemu:///system undefine ${{ parameters.dut }}
fi fi