cb354a5af2
Previously, we set timeout in each step such as Lock testbed, Prepare testbed, Run test and KVM dump. When some issue suck like retry happens in one step, it will cause timeout error, but actually, it only needs more time to success. In this pr, we remove the timeout limit in each step and control the timeout outside in each job. When the job runs more than four hours, it will be cancelled. Why I did it Previously, we set timeout in each step such as Lock testbed, Prepare testbed, Run test and KVM dump. When some issue suck like retry happens in one step, it will cause timeout error, but actually, it only needs more time to success. In this pr, we remove the timeout limit in each step and control the timeout outside in each job. When the job runs more than four hours, it will be cancelled. How I did it Remove the timeout parameter in each step, and control the timeout outside in each job. How to verify it Set the timeout of one job to 4 hours, and when timeout happens, azure pipeline will cancel this job.
383 lines
12 KiB
YAML
383 lines
12 KiB
YAML
# Starter pipeline
|
|
# Start with a minimal pipeline that you can customize to build and deploy your code.
|
|
# Add steps that build, run tests, deploy, and more:
|
|
# https://aka.ms/yaml
|
|
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- master
|
|
- 202???
|
|
paths:
|
|
exclude:
|
|
- .github
|
|
|
|
pr:
|
|
branches:
|
|
include:
|
|
- master
|
|
- 202???
|
|
paths:
|
|
exclude:
|
|
- .github
|
|
|
|
name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
|
|
|
|
resources:
|
|
repositories:
|
|
- repository: sonic-mgmt
|
|
type: github
|
|
name: sonic-net/sonic-mgmt
|
|
endpoint: sonic-net
|
|
- repository: buildimage
|
|
type: github
|
|
name: sonic-net/sonic-buildimage
|
|
endpoint: sonic-net
|
|
ref: master
|
|
|
|
variables:
|
|
- template: .azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
|
|
- template: .azure-pipelines/template-variables.yml@buildimage
|
|
- name: CACHE_MODE
|
|
value: rcache
|
|
- name: ENABLE_FIPS
|
|
value: y
|
|
|
|
stages:
|
|
- stage: BuildVS
|
|
pool: sonicbld
|
|
jobs:
|
|
- template: .azure-pipelines/azure-pipelines-build.yml
|
|
parameters:
|
|
buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) BUILD_MULTIASIC_KVM=y ${{ variables.VERSION_CONTROL_OPTIONS }}'
|
|
jobGroups:
|
|
- name: vs
|
|
|
|
- stage: Build
|
|
pool: sonicbld
|
|
dependsOn: []
|
|
jobs:
|
|
- template: .azure-pipelines/azure-pipelines-build.yml
|
|
parameters:
|
|
buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
|
|
jobGroups:
|
|
- name: broadcom
|
|
variables:
|
|
swi_image: yes
|
|
- name: mellanox
|
|
- name: marvell-armhf
|
|
pool: sonicbld-armhf
|
|
timeoutInMinutes: 1200
|
|
variables:
|
|
PLATFORM_ARCH: armhf
|
|
|
|
- stage: Test
|
|
dependsOn: BuildVS
|
|
condition: and(succeeded(), and(ne(stageDependencies.BuildVS.outputs['vs.SetVar.SKIP_VSTEST'], 'YES'), in(dependencies.BuildVS.result, 'Succeeded', 'SucceededWithIssues')))
|
|
variables:
|
|
- group: Testbed-Tools
|
|
- name: inventory
|
|
value: veos_vtb
|
|
- name: testbed_file
|
|
value: vtestbed.csv
|
|
|
|
# For every test job:
|
|
# continueOnError: false means it's a required test job and will block merge if it fails
|
|
# continueOnError: true means it's an optional test job and will not block merge even though it fails(unless a required test job depends on its result)
|
|
|
|
jobs:
|
|
- job:
|
|
pool: sonictest
|
|
displayName: "vstest"
|
|
timeoutInMinutes: 60
|
|
steps:
|
|
- template: .azure-pipelines/cleanup.yml
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
displayName: 'Checkout code'
|
|
|
|
- script: |
|
|
sudo rm -rf ../target
|
|
sudo rm -rf ../*.deb
|
|
displayName: "Cleanup"
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
inputs:
|
|
source: specific
|
|
project: build
|
|
pipeline: 9
|
|
artifact: sonic-swss-common.amd64.ubuntu20_04
|
|
runVersion: 'latestFromBranch'
|
|
runBranch: 'refs/heads/master'
|
|
displayName: "Download sonic swss common deb packages"
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
inputs:
|
|
artifact: sonic-buildimage.vs
|
|
displayName: "Download sonic-buildimage.vs artifact"
|
|
|
|
- script: |
|
|
set -x
|
|
sudo apt-get update
|
|
sudo apt-get install libyang0.16 -y
|
|
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb
|
|
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
|
|
sudo docker load -i ../target/docker-sonic-vs.gz
|
|
docker tag docker-sonic-vs:latest docker-sonic-vs:$(Build.BuildNumber)
|
|
username=$(id -un)
|
|
|
|
trap "docker ps; docker images; ip netns list; \
|
|
docker rmi docker-sonic-vs:$(Build.BuildNumber); \
|
|
ip netns list | grep -E [-]srv[0-9]+ | awk '{print $1}' | xargs -I {} sudo ip netns delete {}; \
|
|
sudo chown -R ${username}.${username} .; \
|
|
sudo chown -R ${username}.${username} $(System.DefaultWorkingDirectory)" EXIT
|
|
pushd platform/vs/tests
|
|
sudo py.test -v --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.BuildNumber)
|
|
displayName: "Run vs tests"
|
|
|
|
- task: PublishTestResults@2
|
|
inputs:
|
|
testResultsFiles: '**/tr.xml'
|
|
testRunTitle: vstest
|
|
|
|
- job: t0_part1
|
|
pool: sonictest
|
|
displayName: "kvmtest-t0-part1"
|
|
timeoutInMinutes: 400
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-template.yml
|
|
parameters:
|
|
dut: vlab-01
|
|
tbname: vms-kvm-t0
|
|
ptf_name: ptf_vms6-1
|
|
tbtype: t0
|
|
vmtype: ceos
|
|
section: part-1
|
|
|
|
- job: t0_part2
|
|
pool: sonictest
|
|
displayName: "kvmtest-t0-part2"
|
|
timeoutInMinutes: 400
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-template.yml
|
|
parameters:
|
|
dut: vlab-01
|
|
tbname: vms-kvm-t0
|
|
ptf_name: ptf_vms6-1
|
|
tbtype: t0
|
|
vmtype: ceos
|
|
section: part-2
|
|
|
|
- job: t0_testbedv2
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-t0 by TestbedV2"
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: t0
|
|
MIN_WORKER: $(T0_INSTANCE_NUM)
|
|
MAX_WORKER: $(T0_INSTANCE_NUM)
|
|
|
|
- job: t0_2vlans_testbedv2
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-t0-2vlans by TestbedV2"
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: t0
|
|
TEST_SET: t0-2vlans
|
|
MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM)
|
|
MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM)
|
|
DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
|
|
|
|
- job:
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-t0"
|
|
dependsOn:
|
|
- t0_part1
|
|
- t0_part2
|
|
- t0_testbedv2
|
|
- t0_2vlans_testbedv2
|
|
condition: always()
|
|
continueOnError: false
|
|
variables:
|
|
resultOfPart1: $[ dependencies.t0_part1.result ]
|
|
resultOfPart2: $[ dependencies.t0_part2.result ]
|
|
resultOfT0TestbedV2: $[ dependencies.t0_testbedv2.result ]
|
|
resultOfT02VlansTestbedV2: $[ dependencies.t0_2vlans_testbedv2.result ]
|
|
|
|
steps:
|
|
- script: |
|
|
if [ $(resultOfT0TestbedV2) == "Succeeded" ] && [ $(resultOfT02VlansTestbedV2) == "Succeeded" ]; then
|
|
echo "TestbedV2 t0 passed."
|
|
exit 0
|
|
fi
|
|
|
|
if [ $(resultOfPart1) == "Succeeded" ] && [ $(resultOfPart2) == "Succeeded" ]; then
|
|
echo "Classic t0 jobs(both part1 and part2) passed."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Both classic and TestbedV2 t0 jobs failed! Please check the detailed information. (Any of them passed, t0 will be considered as passed)"
|
|
exit 1
|
|
|
|
|
|
- job: t1_lag_classic
|
|
pool: sonictest-t1-lag
|
|
displayName: "kvmtest-t1-lag classic"
|
|
timeoutInMinutes: 400
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-template.yml
|
|
parameters:
|
|
dut: vlab-03
|
|
tbname: vms-kvm-t1-lag
|
|
ptf_name: ptf_vms6-2
|
|
tbtype: t1-lag
|
|
vmtype: ceos
|
|
|
|
- job: t1_lag_testbedv2
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-t1-lag by TestbedV2"
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: t1-lag
|
|
MIN_WORKER: $(T1_LAG_INSTANCE_NUM)
|
|
MAX_WORKER: $(T1_LAG_INSTANCE_NUM)
|
|
|
|
- job:
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-t1-lag"
|
|
dependsOn:
|
|
- t1_lag_classic
|
|
- t1_lag_testbedv2
|
|
condition: always()
|
|
continueOnError: false
|
|
variables:
|
|
resultOfClassic: $[ dependencies.t1_lag_classic.result ]
|
|
resultOfTestbedV2: $[ dependencies.t1_lag_testbedv2.result ]
|
|
steps:
|
|
- script: |
|
|
if [ $(resultOfClassic) == "Succeeded" ] || [ $(resultOfTestbedV2) == "Succeeded" ]; then
|
|
echo "One or both of t1_lag_classic and t1_lag_testbedv2 passed."
|
|
exit 0
|
|
else
|
|
echo "Both t1_lag_classic and t1_lag_testbedv2 failed! Please check the detailed information."
|
|
exit 1
|
|
fi
|
|
|
|
- job:
|
|
pool: sonictest-sonic-t0
|
|
displayName: "kvmtest-t0-sonic"
|
|
timeoutInMinutes: 360
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES'))
|
|
continueOnError: true
|
|
steps:
|
|
- template: .azure-pipelines/run-test-template.yml
|
|
parameters:
|
|
dut: vlab-02
|
|
tbname: vms-kvm-t0-64-32
|
|
ptf_name: ptf_vms6-1
|
|
tbtype: t0-sonic
|
|
vmtype: vsonic
|
|
|
|
- job:
|
|
pool: sonictest-ma
|
|
displayName: "kvmtest-multi-asic-t1-lag"
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES'))
|
|
continueOnError: true
|
|
steps:
|
|
- template: .azure-pipelines/run-test-template.yml
|
|
parameters:
|
|
dut: vlab-08
|
|
tbname: vms-kvm-four-asic-t1-lag
|
|
ptf_name: ptf_vms6-4
|
|
tbtype: multi-asic-t1-lag-pr
|
|
image: sonic-4asic-vs.img.gz
|
|
|
|
- job: multi_asic_testbedv2
|
|
displayName: "kvmtest-multi-asic-t1-lag by TestbedV2"
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: true
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: t1-8-lag
|
|
TEST_SET: multi-asic-t1-lag
|
|
MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
|
|
MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
|
|
NUM_ASIC: 4
|
|
|
|
- job: dualtor_testbedv2
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
displayName: "kvmtest-dualtor-t0 by TestbedV2"
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: false
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: dualtor
|
|
MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
|
|
MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
|
|
COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
|
|
|
|
- job: sonic_t0_testbedv2
|
|
displayName: "kvmtest-t0-sonic by TestbedV2"
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
timeoutInMinutes: 240
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: true
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: t0-64-32
|
|
MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
|
|
MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
|
|
TEST_SET: t0-sonic
|
|
COMMON_EXTRA_PARAMS: "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"
|
|
VM_TYPE: vsonic
|
|
SPECIFIED_PARAMS: '{\"test_pretest.py\":[\"--completeness_level=confident\",\"--allow_recover\"],\"test_posttest.py\":[\"--completeness_level=confident\",\"--allow_recover\"]}'
|
|
|
|
- job: wan_testbedv2
|
|
displayName: "kvmtest-wan by TestbedV2"
|
|
pool:
|
|
vmImage: 'ubuntu-20.04'
|
|
timeoutInMinutes: 1080
|
|
condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
|
|
continueOnError: true
|
|
steps:
|
|
- template: .azure-pipelines/run-test-scheduler-template.yml
|
|
parameters:
|
|
TOPOLOGY: wan-pub
|
|
MIN_WORKER: $(WAN_INSTANCE_NUM)
|
|
MAX_WORKER: $(WAN_INSTANCE_NUM)
|
|
COMMON_EXTRA_PARAMS: "--skip_sanity "
|