[Build] Support Debian snapshot mirror to improve build stability (#13097)
Why I did it [Build] Support Debian snapshot mirror to improve build stability It is to enhance the reproducible build, supports the Debian snapshot mirror. It guarantees all the docker images using the same Debian mirror snapshot and fixes the temporary build failure which is caused by remote Debain mirror indexes changed during the build. It is also to fix the version conflict issue caused by no fixed versions of some of the Debian packages. How I did it Add a new feature to support the Debian snapshot mirror. How to verify it
This commit is contained in:
parent
22fcc760c4
commit
5e4a866e33
@ -198,6 +198,7 @@ $(shell \
|
|||||||
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
|
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
|
||||||
SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
|
SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
|
||||||
DBGOPT='$(DBGOPT)' \
|
DBGOPT='$(DBGOPT)' \
|
||||||
|
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
|
||||||
scripts/generate_buildinfo_config.sh)
|
scripts/generate_buildinfo_config.sh)
|
||||||
|
|
||||||
# Generate the slave Dockerfile, and prepare build info for it
|
# Generate the slave Dockerfile, and prepare build info for it
|
||||||
@ -533,6 +534,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \
|
|||||||
MIRROR_URLS=$(MIRROR_URLS) \
|
MIRROR_URLS=$(MIRROR_URLS) \
|
||||||
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
|
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
|
||||||
GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
|
GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
|
||||||
|
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
|
||||||
$(SONIC_OVERRIDE_BUILD_VARS)
|
$(SONIC_OVERRIDE_BUILD_VARS)
|
||||||
|
|
||||||
.PHONY: sonic-slave-build sonic-slave-bash init reset
|
.PHONY: sonic-slave-build sonic-slave-bash init reset
|
||||||
|
@ -234,6 +234,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
|
|||||||
# docker: docker base images
|
# docker: docker base images
|
||||||
SONIC_VERSION_CONTROL_COMPONENTS ?= none
|
SONIC_VERSION_CONTROL_COMPONENTS ?= none
|
||||||
|
|
||||||
|
# MIRROR_SNAPSHOT - support mirror snapshot flag
|
||||||
|
MIRROR_SNAPSHOT ?= n
|
||||||
|
|
||||||
# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
|
# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
|
||||||
# cache standard installation package and restored back to avoid the package download every time.
|
# cache standard installation package and restored back to avoid the package download every time.
|
||||||
# SONIC_VERSION_CACHE - Method of deb package caching
|
# SONIC_VERSION_CACHE - Method of deb package caching
|
||||||
|
@ -5,16 +5,40 @@ CONFIG_PATH=$1
|
|||||||
export ARCHITECTURE=$2
|
export ARCHITECTURE=$2
|
||||||
export DISTRIBUTION=$3
|
export DISTRIBUTION=$3
|
||||||
|
|
||||||
|
DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
|
||||||
|
MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
|
||||||
|
[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror
|
||||||
|
|
||||||
# The default mirror urls
|
# The default mirror urls
|
||||||
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
|
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
|
||||||
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
|
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
|
||||||
|
|
||||||
|
|
||||||
# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
|
# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
|
||||||
if [ "$ARCHITECTURE" == "armhf" ]; then
|
if [ "$ARCHITECTURE" == "armhf" ]; then
|
||||||
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
|
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
|
||||||
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
|
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$MIRROR_SNAPSHOT" == y ]; then
|
||||||
|
if [ -f $MIRROR_VERSION_FILE ]; then
|
||||||
|
DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
|
||||||
|
DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
|
||||||
|
elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then
|
||||||
|
DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
|
||||||
|
DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/
|
||||||
|
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/
|
||||||
|
|
||||||
|
mkdir -p target/versions/default
|
||||||
|
if [ ! -f target/versions/default/versions-mirror ]; then
|
||||||
|
echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
|
||||||
|
echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
|
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
|
||||||
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
|
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
|
||||||
|
|
||||||
@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2
|
|||||||
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
|
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
|
||||||
|
|
||||||
MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
|
MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
|
||||||
|
if [ "$MIRROR_SNAPSHOT" == y ]; then
|
||||||
|
# Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
|
||||||
|
sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
|
||||||
|
fi
|
||||||
|
@ -11,3 +11,4 @@ echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS"
|
|||||||
echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
|
echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
|
||||||
echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
|
echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
|
||||||
echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG
|
echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG
|
||||||
|
echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG
|
||||||
|
@ -562,12 +562,13 @@ class VersionBuild:
|
|||||||
module.filter(ctypes=ctypes)
|
module.filter(ctypes=ctypes)
|
||||||
|
|
||||||
def get_default_module(self):
|
def get_default_module(self):
|
||||||
if DEFAULT_MODULE in self.modules:
|
default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, []))
|
||||||
return self.modules[DEFAULT_MODULE]
|
|
||||||
ctypes = self.get_component_types()
|
ctypes = self.get_component_types()
|
||||||
dists = self.get_dists()
|
dists = self.get_dists()
|
||||||
components = []
|
components = []
|
||||||
for ctype in ctypes:
|
for ctype in ctypes:
|
||||||
|
if ctype in DEFAULT_OVERWRITE_COMPONENTS:
|
||||||
|
continue
|
||||||
if ctype == 'deb':
|
if ctype == 'deb':
|
||||||
for dist in dists:
|
for dist in dists:
|
||||||
versions = self._get_versions(ctype, dist)
|
versions = self._get_versions(ctype, dist)
|
||||||
@ -579,7 +580,9 @@ class VersionBuild:
|
|||||||
common_versions = self._get_common_versions(versions)
|
common_versions = self._get_common_versions(versions)
|
||||||
component = Component(self.verbose, common_versions, ctype)
|
component = Component(self.verbose, common_versions, ctype)
|
||||||
components.append(component)
|
components.append(component)
|
||||||
return VersionModule(self.verbose, DEFAULT_MODULE, components)
|
module = VersionModule(self.verbose, DEFAULT_MODULE, components)
|
||||||
|
module.overwrite(default_module, True, True)
|
||||||
|
return module
|
||||||
|
|
||||||
def get_aggregatable_modules(self):
|
def get_aggregatable_modules(self):
|
||||||
modules = {}
|
modules = {}
|
||||||
|
1
slave.mk
1
slave.mk
@ -87,6 +87,7 @@ export CROSS_BUILD_ENVIRON
|
|||||||
export BLDENV
|
export BLDENV
|
||||||
export BUILD_WORKDIR
|
export BUILD_WORKDIR
|
||||||
export GZ_COMPRESS_PROGRAM
|
export GZ_COMPRESS_PROGRAM
|
||||||
|
export MIRROR_SNAPSHOT
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
## Utility rules
|
## Utility rules
|
||||||
|
@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions
|
|||||||
VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb
|
VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb
|
||||||
WEB_VERSION_FILE=$VERSION_PATH/versions-web
|
WEB_VERSION_FILE=$VERSION_PATH/versions-web
|
||||||
BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
|
BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
|
||||||
REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
|
REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
|
||||||
DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
|
DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
|
||||||
|
|
||||||
. $BUILDINFO_PATH/config/buildinfo.config
|
. $BUILDINFO_PATH/config/buildinfo.config
|
||||||
@ -112,14 +112,39 @@ set_reproducible_mirrors()
|
|||||||
{
|
{
|
||||||
# Remove the charater # in front of the line if matched
|
# Remove the charater # in front of the line if matched
|
||||||
local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/"
|
local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/"
|
||||||
|
# Add the character # in front of the line, if not match the URL pattern condition
|
||||||
|
local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/"
|
||||||
|
local expression3="\$a#SET_REPR_MIRRORS"
|
||||||
if [ "$1" = "-d" ]; then
|
if [ "$1" = "-d" ]; then
|
||||||
# Add the charater # in front of the line if match
|
# Add the charater # in front of the line if match
|
||||||
expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/"
|
expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/"
|
||||||
|
# Remove the character # in front of the line, if not match the URL pattern condition
|
||||||
|
expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/"
|
||||||
|
expression3="/#SET_REPR_MIRRORS/d"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)"
|
local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)"
|
||||||
for mirror in $mirrors; do
|
for mirror in $mirrors; do
|
||||||
|
if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Make sure no duplicate operations on the mirror config file
|
||||||
|
if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") ||
|
||||||
|
([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Enable or disable the reproducible mirrors
|
||||||
$SUDO sed -i "$expression" "$mirror"
|
$SUDO sed -i "$expression" "$mirror"
|
||||||
|
|
||||||
|
# Enable or disable the none reproducible mirrors
|
||||||
|
if [ "$MIRROR_SNAPSHOT" == y ]; then
|
||||||
|
$SUDO sed -ri "$expression2" "$mirror"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Add or remove the SET_REPR_MIRRORS flag
|
||||||
|
$SUDO sed -i "$expression3" "$mirror"
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -25,6 +25,13 @@ dpkg-query -W -f '${Package}==${Version}\n' | grep -Ev "${SKIP_VERSION_PACKAGE}"
|
|||||||
## Add the the packages purged
|
## Add the the packages purged
|
||||||
[ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
|
[ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
|
||||||
|
|
||||||
|
## Add mirror versions
|
||||||
|
while read -r line; do
|
||||||
|
mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//")
|
||||||
|
date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ)
|
||||||
|
echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror
|
||||||
|
done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null)
|
||||||
|
|
||||||
## Print the unique and sorted result
|
## Print the unique and sorted result
|
||||||
sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
|
sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
|
||||||
if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then
|
if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then
|
||||||
@ -33,5 +40,8 @@ fi
|
|||||||
if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then
|
if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then
|
||||||
sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
|
sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
|
||||||
fi
|
fi
|
||||||
|
if [ -e "${TARGET_PATH}/versions-mirror" ]; then
|
||||||
|
sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror"
|
||||||
|
fi
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
|
Loading…
Reference in New Issue
Block a user