diff --git a/.gitignore b/.gitignore
index 406b8038d7..5797d0cd11 100644
--- a/.gitignore
+++ b/.gitignore
@@ -105,3 +105,6 @@ htmlcov/
 # Debian mirror Sources
 sources.list.*
 !sources.list*.j2
+
+# Generated mirror configs
+apt-retries-count
diff --git a/build_debian.sh b/build_debian.sh
index a8c44b32cb..505806fdc1 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -113,7 +113,8 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
 ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
 scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
 sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
-sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until,apt-multiple-retries} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
+sudo cp files/apt/apt-retries-count $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
+sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
 
 ## Note: set lang to prevent locale warnings in your chroot
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y update
diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2
index 3b76ef6917..c05973510f 100644
--- a/dockers/docker-base-buster/Dockerfile.j2
+++ b/dockers/docker-base-buster/Dockerfile.j2
@@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 
 # Update apt cache and
 # pre-install fundamental packages
diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2
index 5db96e37ba..8963024d17 100644
--- a/dockers/docker-base-stretch/Dockerfile.j2
+++ b/dockers/docker-base-stretch/Dockerfile.j2
@@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 
 # Update apt cache and
 # pre-install fundamental packages
diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2
index cd839e6075..66fab19016 100644
--- a/dockers/docker-base/Dockerfile.j2
+++ b/dockers/docker-base/Dockerfile.j2
@@ -38,6 +38,7 @@ COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
 COPY ["sources.list", "/etc/apt/sources.list"]
 {% endif %}
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 RUN apt-get update
 
 # Pre-install fundamental packages
diff --git a/files/apt/apt-retries-count.j2 b/files/apt/apt-retries-count.j2
new file mode 100644
index 0000000000..8ac0a7e386
--- /dev/null
+++ b/files/apt/apt-retries-count.j2
@@ -0,0 +1,2 @@
+# Instruct apt to retry downloads on failures
+Acquire::Retries "{{ APT_RETRIES_COUNT }}";
diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh
index aee56f23ae..1ee92d4839 100755
--- a/scripts/build_mirror_config.sh
+++ b/scripts/build_mirror_config.sh
@@ -5,6 +5,10 @@ CONFIG_PATH=$1
 export ARCHITECTURE=$2
 export DISTRIBUTION=$3
 
+# Handling default
+[[ -z $APT_RETRIES_COUNT ]] && APT_RETRIES_COUNT=20
+export APT_RETRIES_COUNT
+
 DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
 MIRROR_VERSION_FILE=
 [[ "$SONIC_VERSION_CONTROL_COMPONENTS" == *deb* || $SONIC_VERSION_CONTROL_COMPONENTS == *all* ]] && MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
@@ -40,6 +44,7 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then
     fi
 fi
 
+# Handle sources list
 [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
 [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
 
@@ -53,3 +58,8 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then
     # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
     sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
 fi
+
+# Handle apt retry count config
+APT_RETRIES_COUNT_FILENAME=apt-retries-count
+TEMPLATE=files/apt/$APT_RETRIES_COUNT_FILENAME.j2
+j2 $TEMPLATE > $CONFIG_PATH/$APT_RETRIES_COUNT_FILENAME
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index 06d64fc0ba..78f7deaf65 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -22,7 +22,7 @@ FROM {{ prefix }}debian:bullseye
 MAINTAINER gulv@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
-COPY ["apt-multiple-retries", "/etc/apt/apt.conf.d/"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
diff --git a/sonic-slave-bullseye/apt-multiple-retries b/sonic-slave-bullseye/apt-multiple-retries
deleted file mode 100644
index c4c638252f..0000000000
--- a/sonic-slave-bullseye/apt-multiple-retries
+++ /dev/null
@@ -1,4 +0,0 @@
-# Instruct apt to retry downloads on failures
-# This is required only for bullseye.
-
-Acquire::Retries "3";
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index cfa444347d..bb1ec2c970 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -21,6 +21,7 @@ FROM {{ prefix }}debian:buster
 MAINTAINER gulv@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
diff --git a/sonic-slave-jessie/Dockerfile.j2 b/sonic-slave-jessie/Dockerfile.j2
index 59bc7300dd..96ac4653e4 100644
--- a/sonic-slave-jessie/Dockerfile.j2
+++ b/sonic-slave-jessie/Dockerfile.j2
@@ -10,6 +10,7 @@ FROM {{ prefix }}debian:jessie
 MAINTAINER johnar@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 
 ## Remove retired jessie-updates repo
 RUN sed -i '/http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2
index be7744b74d..d63861c6d9 100644
--- a/sonic-slave-stretch/Dockerfile.j2
+++ b/sonic-slave-stretch/Dockerfile.j2
@@ -10,6 +10,7 @@ FROM {{ prefix }}debian:stretch
 MAINTAINER gulv@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
+COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 
 ## Make apt-get non-interactive
diff --git a/src/sonic-build-hooks/hooks/curl b/src/sonic-build-hooks/hooks/curl
index 5e8dd91b88..72e9dd7176 100755
--- a/src/sonic-build-hooks/hooks/curl
+++ b/src/sonic-build-hooks/hooks/curl
@@ -3,6 +3,19 @@
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 [ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/curl
 
+# Retry if something super-weird has happened. Use --retry-connrefused and
+# --retry options for curl.
+# --retry-connrefused - Consider "connection refused" a transient error and try
+# again. Normally wget/curl gives up on a URL when it is unable to connect to
+# the site because failure to connect is taken as a sign that the server is not
+# running at all and that retries would not help. This option is for mirroring
+# unreliable sites whose servers tend to disappear for short periods of time.
+# --retry - If a transient error is returned when curl tries to perform a
+# transfer, it will retry this number of times before giving up. Transient error
+# means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500,
+# 502, 503 or 504 response code.
+REAL_COMMAND="$REAL_COMMAND --retry-connrefused --retry 5"
+
 if [ "$SKIP_BUILD_HOOK" == y ]; then
     $REAL_COMMAND "$@"
     exit $?
diff --git a/src/sonic-build-hooks/hooks/wget b/src/sonic-build-hooks/hooks/wget
index c4cb1a3d1b..3ea90f8ecf 100755
--- a/src/sonic-build-hooks/hooks/wget
+++ b/src/sonic-build-hooks/hooks/wget
@@ -7,6 +7,15 @@ if [ -z "$REAL_COMMAND" ]; then
     exit 1
 fi
 
+# Retry if something super-weird has happened. Use --retry-connrefused option
+# for wget.
+# --retry-connrefused - Consider "connection refused" a transient error and try
+# again. Normally wget/curl gives up on a URL when it is unable to connect to
+# the site because failure to connect is taken as a sign that the server is not
+# running at all and that retries would not help. This option is for mirroring
+# unreliable sites whose servers tend to disappear for short periods of time.
+REAL_COMMAND="$REAL_COMMAND --retry-connrefused"
+
 if [ "$SKIP_BUILD_HOOK" == y ]; then
     $REAL_COMMAND "$@"
     exit $?
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
index 44832e4c71..2c85cb665c 100755
--- a/src/sonic-build-hooks/scripts/buildinfo_base.sh
+++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -13,6 +13,7 @@ WEB_VERSION_FILE=$VERSION_PATH/versions-web
 BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
 REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
 DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
+GET_RETRY_COUNT=5
 
 . $BUILDINFO_PATH/config/buildinfo.config
 
@@ -242,10 +243,17 @@ download_packages()
         return $result
     fi
 
-    $REAL_COMMAND "${parameters[@]}"
-    result=$?
+    # Retry if something super-weird has happened
+    for ((i = 1; i <= GET_RETRY_COUNT; i++)); do
+        $REAL_COMMAND "${parameters[@]}"
+        result=$?
+        if [ $result -eq 0 ]; then
+            break
+        fi
+        log_err "Try $i: $REAL_COMMAND failed to get: ${parameters[@]}. Retry.."
+    done
 
-    #Return if there is any error
+    # Return if there is any error
     if [ ${result} -ne 0 ]; then
         exit ${result}
     fi