[ci] Kill hanged docker build process to avoid build timeout issue. (#13729)
Why I did it Docker build has a low rate of hanging up. It hangs on different steps. So, it looks like a bug in docker daemon. How I did it Start a daemon process to scan running time more than 1 hours, and kill the process.
This commit is contained in:
parent
868a1d8e39
commit
648ce4b12d
@ -102,6 +102,25 @@ jobs:
|
|||||||
buildSteps:
|
buildSteps:
|
||||||
- template: template-skipvstest.yml
|
- template: template-skipvstest.yml
|
||||||
- bash: |
|
- bash: |
|
||||||
|
(
|
||||||
|
while true
|
||||||
|
do
|
||||||
|
sleep 120
|
||||||
|
now=$(date +%s)
|
||||||
|
pids=$(ps aux | grep -v grep | grep -E "^.{,100}docker build" | awk '{print$2}')
|
||||||
|
for pid in $pids
|
||||||
|
do
|
||||||
|
start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
|
||||||
|
time_s=$(($now-$start))
|
||||||
|
if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
|
||||||
|
echo =========== $(date +%F%T) $time_s &>> target/daemon.log
|
||||||
|
ps $pid &>> target/daemon.log
|
||||||
|
sudo kill $pid
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
daemon_pid=$!
|
||||||
set -ex
|
set -ex
|
||||||
if [ $(GROUP_NAME) == vs ]; then
|
if [ $(GROUP_NAME) == vs ]; then
|
||||||
if [ $(dbg_image) == yes ]; then
|
if [ $(dbg_image) == yes ]; then
|
||||||
@ -164,5 +183,6 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
|
make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
|
||||||
fi
|
fi
|
||||||
|
sudo kill $daemon_pid
|
||||||
displayName: "Build sonic image"
|
displayName: "Build sonic image"
|
||||||
- template: check-dirty-version.yml
|
- template: check-dirty-version.yml
|
||||||
|
@ -4,3 +4,5 @@ variables:
|
|||||||
SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
|
SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
|
||||||
SONIC_BUILD_RETRY_COUNT: 3
|
SONIC_BUILD_RETRY_COUNT: 3
|
||||||
SONIC_BUILD_RETRY_INTERVAL: 600
|
SONIC_BUILD_RETRY_INTERVAL: 600
|
||||||
|
DOCKER_BUILDKIT: 0
|
||||||
|
DOCKER_BUILD_TIMEOUT: 3600
|
||||||
|
Reference in New Issue
Block a user