From 648ce4b12d63f662367c84373b886eb51f786b48 Mon Sep 17 00:00:00 2001
From: Liu Shilong <shilongliu@microsoft.com>
Date: Sat, 11 Feb 2023 01:14:28 +0800
Subject: [PATCH] [ci] Kill hanged docker build process to avoid build timeout
 issue. (#13729)

Why I did it
Docker build has a low rate of hanging up.
It hangs on different steps. So, it looks like a bug in docker daemon.

How I did it
Start a daemon process to scan running time more than 1 hours, and kill the process.
---
 .azure-pipelines/azure-pipelines-build.yml | 20 ++++++++++++++++++++
 .azure-pipelines/template-variables.yml    |  2 ++
 2 files changed, 22 insertions(+)

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index 68ce08fa41..51ad919817 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -102,6 +102,25 @@ jobs:
     buildSteps:
       - template: template-skipvstest.yml
       - bash: |
+          (
+            while true
+            do
+              sleep 120
+              now=$(date +%s)
+              pids=$(ps aux | grep -v grep | grep -E "^.{,100}docker build" | awk '{print$2}')
+              for pid in $pids
+              do
+                start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+                time_s=$(($now-$start))
+                if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+                  echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+                  ps $pid &>> target/daemon.log
+                  sudo kill $pid
+                fi
+              done
+            done
+          ) &
+          daemon_pid=$!
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
@@ -164,5 +183,6 @@ jobs:
             fi
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
+          sudo kill $daemon_pid
         displayName: "Build sonic image"
       - template: check-dirty-version.yml
diff --git a/.azure-pipelines/template-variables.yml b/.azure-pipelines/template-variables.yml
index 027dfee998..42d46182d2 100644
--- a/.azure-pipelines/template-variables.yml
+++ b/.azure-pipelines/template-variables.yml
@@ -4,3 +4,5 @@ variables:
   SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
   SONIC_BUILD_RETRY_COUNT: 3
   SONIC_BUILD_RETRY_INTERVAL: 600
+  DOCKER_BUILDKIT: 0
+  DOCKER_BUILD_TIMEOUT: 3600