sonic-buildimage/scripts/build_mirror_config.sh

66 lines
3.4 KiB
Bash
Raw Normal View History

#!/bin/bash
# Generate the sources.list.<arch> in the config path
CONFIG_PATH=$1
export ARCHITECTURE=$2
export DISTRIBUTION=$3
Improve remote fetch (#12795) ### Why I did it To fix those errors: One: ``` Connecting to urm.nvidia.com (urm.nvidia.com)|*.*.*.*|:443... connected. GnuTLS: Error in the pull function. Unable to establish SSL connection. Error 4 make[1]: Leaving directory '/sonic/src/smartmontools' [ target/debs/bullseye/smartmontools_6.6-1_amd64.deb ] ``` Second: ``` Get:90 https://debian-mirror-url buster/main amd64 librrd-dev amd64 1.7.1-2 [284 kB] Get:91 https://debian-mirror-url buster/main amd64 psmisc amd64 23.2-1+deb10u1 [126 kB] Get:92 https://debian-mirror-url buster/main amd64 python-smbus amd64 4.1-1 [12.2 kB] Get:93 https://debian-mirror-url buster/main amd64 python3.7-dev amd64 3.7.3-2+deb10u3 [510 kB] Get:94 https://debian-mirror-url buster/main amd64 python3-dev amd64 3.7.3-1 [1264 B] Get:95 https://debian-mirror-url buster/main amd64 python3-smbus amd64 4.1-1 [12.5 kB] Get:96 https://debian-mirror-url buster/main amd64 rrdtool amd64 1.7.1-2 [485 kB] Fetched 122 MB in 12s (9976 kB/s) E: Failed to fetch https://debian-mirror-url/pool/main/p/python-defaults/python2-minimal_2.7.16-1_amd64.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Failed to fetch https://debian-mirror-url/pool/main/f/fontconfig/fontconfig-config_2.13.1-2_all.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? The command '/bin/sh -c apt-get update && apt-get install -y build-essential python3-dev ipmitool librrd8 librrd-dev rrdtool python-smbus python3-smbus dmidecode i2c-tools psmisc libpci3' returned a non-zero code: 100 [ target/docker-platform-monitor.gz ] Error 1 ``` #### How I did it Add retry mechanism to apt, wget, and curl hooks
2023-09-23 20:07:04 -05:00
# Handling default
[[ -z $APT_RETRIES_COUNT ]] && APT_RETRIES_COUNT=20
export APT_RETRIES_COUNT
DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
MIRROR_VERSION_FILE=
[[ "$SONIC_VERSION_CONTROL_COMPONENTS" == *deb* || $SONIC_VERSION_CONTROL_COMPONENTS == *all* ]] && MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror
# The default mirror urls
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
if [ "$ARCHITECTURE" == "armhf" ]; then
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
fi
if [ "$MIRROR_SNAPSHOT" == y ]; then
if [ -f "$MIRROR_VERSION_FILE" ]; then
DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then
DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
fi
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/
mkdir -p target/versions/default
if [ ! -f target/versions/default/versions-mirror ]; then
echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
fi
fi
Improve remote fetch (#12795) ### Why I did it To fix those errors: One: ``` Connecting to urm.nvidia.com (urm.nvidia.com)|*.*.*.*|:443... connected. GnuTLS: Error in the pull function. Unable to establish SSL connection. Error 4 make[1]: Leaving directory '/sonic/src/smartmontools' [ target/debs/bullseye/smartmontools_6.6-1_amd64.deb ] ``` Second: ``` Get:90 https://debian-mirror-url buster/main amd64 librrd-dev amd64 1.7.1-2 [284 kB] Get:91 https://debian-mirror-url buster/main amd64 psmisc amd64 23.2-1+deb10u1 [126 kB] Get:92 https://debian-mirror-url buster/main amd64 python-smbus amd64 4.1-1 [12.2 kB] Get:93 https://debian-mirror-url buster/main amd64 python3.7-dev amd64 3.7.3-2+deb10u3 [510 kB] Get:94 https://debian-mirror-url buster/main amd64 python3-dev amd64 3.7.3-1 [1264 B] Get:95 https://debian-mirror-url buster/main amd64 python3-smbus amd64 4.1-1 [12.5 kB] Get:96 https://debian-mirror-url buster/main amd64 rrdtool amd64 1.7.1-2 [485 kB] Fetched 122 MB in 12s (9976 kB/s) E: Failed to fetch https://debian-mirror-url/pool/main/p/python-defaults/python2-minimal_2.7.16-1_amd64.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Failed to fetch https://debian-mirror-url/pool/main/f/fontconfig/fontconfig-config_2.13.1-2_all.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? The command '/bin/sh -c apt-get update && apt-get install -y build-essential python3-dev ipmitool librrd8 librrd-dev rrdtool python-smbus python3-smbus dmidecode i2c-tools psmisc libpci3' returned a non-zero code: 100 [ target/docker-platform-monitor.gz ] Error 1 ``` #### How I did it Add retry mechanism to apt, wget, and curl hooks
2023-09-23 20:07:04 -05:00
# Handle sources list
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
TEMPLATE=files/apt/sources.list.j2
[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2
[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
if [ "$MIRROR_SNAPSHOT" == y ]; then
# Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
fi
Improve remote fetch (#12795) ### Why I did it To fix those errors: One: ``` Connecting to urm.nvidia.com (urm.nvidia.com)|*.*.*.*|:443... connected. GnuTLS: Error in the pull function. Unable to establish SSL connection. Error 4 make[1]: Leaving directory '/sonic/src/smartmontools' [ target/debs/bullseye/smartmontools_6.6-1_amd64.deb ] ``` Second: ``` Get:90 https://debian-mirror-url buster/main amd64 librrd-dev amd64 1.7.1-2 [284 kB] Get:91 https://debian-mirror-url buster/main amd64 psmisc amd64 23.2-1+deb10u1 [126 kB] Get:92 https://debian-mirror-url buster/main amd64 python-smbus amd64 4.1-1 [12.2 kB] Get:93 https://debian-mirror-url buster/main amd64 python3.7-dev amd64 3.7.3-2+deb10u3 [510 kB] Get:94 https://debian-mirror-url buster/main amd64 python3-dev amd64 3.7.3-1 [1264 B] Get:95 https://debian-mirror-url buster/main amd64 python3-smbus amd64 4.1-1 [12.5 kB] Get:96 https://debian-mirror-url buster/main amd64 rrdtool amd64 1.7.1-2 [485 kB] Fetched 122 MB in 12s (9976 kB/s) E: Failed to fetch https://debian-mirror-url/pool/main/p/python-defaults/python2-minimal_2.7.16-1_amd64.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Failed to fetch https://debian-mirror-url/pool/main/f/fontconfig/fontconfig-config_2.13.1-2_all.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? The command '/bin/sh -c apt-get update && apt-get install -y build-essential python3-dev ipmitool librrd8 librrd-dev rrdtool python-smbus python3-smbus dmidecode i2c-tools psmisc libpci3' returned a non-zero code: 100 [ target/docker-platform-monitor.gz ] Error 1 ``` #### How I did it Add retry mechanism to apt, wget, and curl hooks
2023-09-23 20:07:04 -05:00
# Handle apt retry count config
APT_RETRIES_COUNT_FILENAME=apt-retries-count
TEMPLATE=files/apt/$APT_RETRIES_COUNT_FILENAME.j2
j2 $TEMPLATE > $CONFIG_PATH/$APT_RETRIES_COUNT_FILENAME