From 52f6dd65a379f870b7098a4f2dbb6ede52f89e5d Mon Sep 17 00:00:00 2001 From: Yevhen Fastiuk Date: Sun, 24 Sep 2023 04:07:04 +0300 Subject: [PATCH] Improve remote fetch (#12795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Why I did it To fix those errors: One: ``` Connecting to urm.nvidia.com (urm.nvidia.com)|*.*.*.*|:443... connected. GnuTLS: Error in the pull function. Unable to establish SSL connection. Error 4 make[1]: Leaving directory '/sonic/src/smartmontools' [ target/debs/bullseye/smartmontools_6.6-1_amd64.deb ] ``` Second: ``` Get:90 https://debian-mirror-url buster/main amd64 librrd-dev amd64 1.7.1-2 [284 kB] Get:91 https://debian-mirror-url buster/main amd64 psmisc amd64 23.2-1+deb10u1 [126 kB] Get:92 https://debian-mirror-url buster/main amd64 python-smbus amd64 4.1-1 [12.2 kB] Get:93 https://debian-mirror-url buster/main amd64 python3.7-dev amd64 3.7.3-2+deb10u3 [510 kB] Get:94 https://debian-mirror-url buster/main amd64 python3-dev amd64 3.7.3-1 [1264 B] Get:95 https://debian-mirror-url buster/main amd64 python3-smbus amd64 4.1-1 [12.5 kB] Get:96 https://debian-mirror-url buster/main amd64 rrdtool amd64 1.7.1-2 [485 kB] Fetched 122 MB in 12s (9976 kB/s) E: Failed to fetch https://debian-mirror-url/pool/main/p/python-defaults/python2-minimal_2.7.16-1_amd64.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Failed to fetch https://debian-mirror-url/pool/main/f/fontconfig/fontconfig-config_2.13.1-2_all.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? The command '/bin/sh -c apt-get update && apt-get install -y build-essential python3-dev ipmitool librrd8 librrd-dev rrdtool python-smbus python3-smbus dmidecode i2c-tools psmisc libpci3' returned a non-zero code: 100 [ target/docker-platform-monitor.gz ] Error 1 ``` #### How I did it Add retry mechanism to apt, wget, and curl hooks --- .gitignore | 3 +++ build_debian.sh | 3 ++- dockers/docker-base-buster/Dockerfile.j2 | 1 + dockers/docker-base-stretch/Dockerfile.j2 | 1 + dockers/docker-base/Dockerfile.j2 | 1 + files/apt/apt-retries-count.j2 | 2 ++ scripts/build_mirror_config.sh | 10 ++++++++++ sonic-slave-bullseye/Dockerfile.j2 | 2 +- sonic-slave-bullseye/apt-multiple-retries | 4 ---- sonic-slave-buster/Dockerfile.j2 | 1 + sonic-slave-jessie/Dockerfile.j2 | 1 + sonic-slave-stretch/Dockerfile.j2 | 1 + src/sonic-build-hooks/hooks/curl | 13 +++++++++++++ src/sonic-build-hooks/hooks/wget | 9 +++++++++ src/sonic-build-hooks/scripts/buildinfo_base.sh | 14 +++++++++++--- 15 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 files/apt/apt-retries-count.j2 delete mode 100644 sonic-slave-bullseye/apt-multiple-retries diff --git a/.gitignore b/.gitignore index 406b8038d7..5797d0cd11 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,6 @@ htmlcov/ # Debian mirror Sources sources.list.* !sources.list*.j2 + +# Generated mirror configs +apt-retries-count diff --git a/build_debian.sh b/build_debian.sh index a8c44b32cb..505806fdc1 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -113,7 +113,8 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list -sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until,apt-multiple-retries} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ +sudo cp files/apt/apt-retries-count $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ +sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ ## Note: set lang to prevent locale warnings in your chroot sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y update diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index 3b76ef6917..c05973510f 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] # Update apt cache and # pre-install fundamental packages diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2 index 5db96e37ba..8963024d17 100644 --- a/dockers/docker-base-stretch/Dockerfile.j2 +++ b/dockers/docker-base-stretch/Dockerfile.j2 @@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] # Update apt cache and # pre-install fundamental packages diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2 index cd839e6075..66fab19016 100644 --- a/dockers/docker-base/Dockerfile.j2 +++ b/dockers/docker-base/Dockerfile.j2 @@ -38,6 +38,7 @@ COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] COPY ["sources.list", "/etc/apt/sources.list"] {% endif %} COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] RUN apt-get update # Pre-install fundamental packages diff --git a/files/apt/apt-retries-count.j2 b/files/apt/apt-retries-count.j2 new file mode 100644 index 0000000000..8ac0a7e386 --- /dev/null +++ b/files/apt/apt-retries-count.j2 @@ -0,0 +1,2 @@ +# Instruct apt to retry downloads on failures +Acquire::Retries "{{ APT_RETRIES_COUNT }}"; diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh index aee56f23ae..1ee92d4839 100755 --- a/scripts/build_mirror_config.sh +++ b/scripts/build_mirror_config.sh @@ -5,6 +5,10 @@ CONFIG_PATH=$1 export ARCHITECTURE=$2 export DISTRIBUTION=$3 +# Handling default +[[ -z $APT_RETRIES_COUNT ]] && APT_RETRIES_COUNT=20 +export APT_RETRIES_COUNT + DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net MIRROR_VERSION_FILE= [[ "$SONIC_VERSION_CONTROL_COMPONENTS" == *deb* || $SONIC_VERSION_CONTROL_COMPONENTS == *all* ]] && MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror @@ -40,6 +44,7 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then fi fi +# Handle sources list [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS @@ -53,3 +58,8 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE fi + +# Handle apt retry count config +APT_RETRIES_COUNT_FILENAME=apt-retries-count +TEMPLATE=files/apt/$APT_RETRIES_COUNT_FILENAME.j2 +j2 $TEMPLATE > $CONFIG_PATH/$APT_RETRIES_COUNT_FILENAME diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index 06d64fc0ba..78f7deaf65 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -22,7 +22,7 @@ FROM {{ prefix }}debian:bullseye MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] -COPY ["apt-multiple-retries", "/etc/apt/apt.conf.d/"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] {%- if CROSS_BUILD_ENVIRON != "y" %} COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] diff --git a/sonic-slave-bullseye/apt-multiple-retries b/sonic-slave-bullseye/apt-multiple-retries deleted file mode 100644 index c4c638252f..0000000000 --- a/sonic-slave-bullseye/apt-multiple-retries +++ /dev/null @@ -1,4 +0,0 @@ -# Instruct apt to retry downloads on failures -# This is required only for bullseye. - -Acquire::Retries "3"; diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index cfa444347d..bb1ec2c970 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -21,6 +21,7 @@ FROM {{ prefix }}debian:buster MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] {%- if CROSS_BUILD_ENVIRON != "y" %} COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] diff --git a/sonic-slave-jessie/Dockerfile.j2 b/sonic-slave-jessie/Dockerfile.j2 index 59bc7300dd..96ac4653e4 100644 --- a/sonic-slave-jessie/Dockerfile.j2 +++ b/sonic-slave-jessie/Dockerfile.j2 @@ -10,6 +10,7 @@ FROM {{ prefix }}debian:jessie MAINTAINER johnar@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] ## Remove retired jessie-updates repo RUN sed -i '/http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index be7744b74d..d63861c6d9 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -10,6 +10,7 @@ FROM {{ prefix }}debian:stretch MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] +COPY ["apt-retries-count", "/etc/apt/apt.conf.d"] COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] ## Make apt-get non-interactive diff --git a/src/sonic-build-hooks/hooks/curl b/src/sonic-build-hooks/hooks/curl index 5e8dd91b88..72e9dd7176 100755 --- a/src/sonic-build-hooks/hooks/curl +++ b/src/sonic-build-hooks/hooks/curl @@ -3,6 +3,19 @@ . /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/curl +# Retry if something super-weird has happened. Use --retry-connrefused and +# --retry options for curl. +# --retry-connrefused - Consider "connection refused" a transient error and try +# again. Normally wget/curl gives up on a URL when it is unable to connect to +# the site because failure to connect is taken as a sign that the server is not +# running at all and that retries would not help. This option is for mirroring +# unreliable sites whose servers tend to disappear for short periods of time. +# --retry - If a transient error is returned when curl tries to perform a +# transfer, it will retry this number of times before giving up. Transient error +# means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, +# 502, 503 or 504 response code. +REAL_COMMAND="$REAL_COMMAND --retry-connrefused --retry 5" + if [ "$SKIP_BUILD_HOOK" == y ]; then $REAL_COMMAND "$@" exit $? diff --git a/src/sonic-build-hooks/hooks/wget b/src/sonic-build-hooks/hooks/wget index c4cb1a3d1b..3ea90f8ecf 100755 --- a/src/sonic-build-hooks/hooks/wget +++ b/src/sonic-build-hooks/hooks/wget @@ -7,6 +7,15 @@ if [ -z "$REAL_COMMAND" ]; then exit 1 fi +# Retry if something super-weird has happened. Use --retry-connrefused option +# for wget. +# --retry-connrefused - Consider "connection refused" a transient error and try +# again. Normally wget/curl gives up on a URL when it is unable to connect to +# the site because failure to connect is taken as a sign that the server is not +# running at all and that retries would not help. This option is for mirroring +# unreliable sites whose servers tend to disappear for short periods of time. +REAL_COMMAND="$REAL_COMMAND --retry-connrefused" + if [ "$SKIP_BUILD_HOOK" == y ]; then $REAL_COMMAND "$@" exit $? diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index 44832e4c71..2c85cb665c 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -13,6 +13,7 @@ WEB_VERSION_FILE=$VERSION_PATH/versions-web BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/' DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock +GET_RETRY_COUNT=5 . $BUILDINFO_PATH/config/buildinfo.config @@ -242,10 +243,17 @@ download_packages() return $result fi - $REAL_COMMAND "${parameters[@]}" - result=$? + # Retry if something super-weird has happened + for ((i = 1; i <= GET_RETRY_COUNT; i++)); do + $REAL_COMMAND "${parameters[@]}" + result=$? + if [ $result -eq 0 ]; then + break + fi + log_err "Try $i: $REAL_COMMAND failed to get: ${parameters[@]}. Retry.." + done - #Return if there is any error + # Return if there is any error if [ ${result} -ne 0 ]; then exit ${result} fi