diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index 09a46573fe..ba3227a349 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -109,6 +109,7 @@ jobs: buildSteps: - template: template-skipvstest.yml + - template: template-daemon.yml - bash: | set -ex if [ $(GROUP_NAME) == vs ]; then diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 6569c8ed0b..4f74de0f41 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -1,5 +1,11 @@ steps: - script: | + set -x + # kill daemon process + ps $(cat /tmp/azp_daemon_kill_docker_pid) + sudo kill $(cat /tmp/azp_daemon_kill_docker_pid) + rm /tmp/azp_daemon_kill_docker_pid + if sudo [ -f /var/run/march/docker.pid ] ; then pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid fi @@ -11,4 +17,5 @@ steps: pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid fi sudo rm -rf $(ls -A1) + condition: always() displayName: "Clean Workspace" diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml new file mode 100644 index 0000000000..b0915557ce --- /dev/null +++ b/.azure-pipelines/template-daemon.yml @@ -0,0 +1,24 @@ +steps: +- bash: | + ( + while true + do + sleep 120 + now=$(date +%s) + pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f1) + for pid in $pids + do + start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s) + time_s=$(($now-$start)) + if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then + echo =========== $(date +%F%T) $time_s &>> target/daemon.log + ps $pid &>> target/daemon.log + sudo kill $pid + fi + done + done + ) & + daemon_pid=$! + ps $daemon_pid + echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid + displayName: start daemon to kill hang docker diff --git a/.azure-pipelines/template-variables.yml b/.azure-pipelines/template-variables.yml index ce2ec68743..42d46182d2 100644 --- a/.azure-pipelines/template-variables.yml +++ b/.azure-pipelines/template-variables.yml @@ -5,3 +5,4 @@ variables: SONIC_BUILD_RETRY_COUNT: 3 SONIC_BUILD_RETRY_INTERVAL: 600 DOCKER_BUILDKIT: 0 + DOCKER_BUILD_TIMEOUT: 3600