From c9806ec3c39864cf6e9dc3244154a69a4f5b342a Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Sat, 11 Feb 2023 01:33:54 +0800
Subject: [PATCH] [Build][202211] Support Debian snapshot mirror to improve
 build stability (#13371) (#13382)

Why I did it
Cherry pick from #13097
[Build] Support Debian snapshot mirror to improve build stability

It is to enhance the reproducible build, supports the Debian snapshot mirror. It guarantees all the docker images using the same Debian mirror snapshot and fixes the temporary build failure which is caused by remote Debain mirror indexes changed during the build. It is also to fix the version conflict issue caused by no fixed versions of some of the Debian packages.

How I did it
Add a new feature to support the Debian snapshot mirror.

How to verify it
---
 Makefile.work                                 | 10 +++++--
 rules/config                                  |  3 ++
 scripts/build_mirror_config.sh                | 28 +++++++++++++++++++
 scripts/generate_buildinfo_config.sh          |  1 +
 scripts/versions_manager.py                   |  9 ++++--
 slave.mk                                      |  1 +
 .../scripts/buildinfo_base.sh                 | 27 +++++++++++++++++-
 .../scripts/collect_version_files             | 10 +++++++
 8 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/Makefile.work b/Makefile.work
index 7466a88882..e5a6714af0 100644
--- a/Makefile.work
+++ b/Makefile.work
@@ -153,9 +153,12 @@ endif
 endif
 
 # Generate the version control build info
-$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
-    TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
-    scripts/generate_buildinfo_config.sh)
+$(shell \
+	SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
+	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
+	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+	MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
+	scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
@@ -388,6 +391,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \
                            MIRROR_URLS=$(MIRROR_URLS) \
                            MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
+                           MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
diff --git a/rules/config b/rules/config
index 56c3946ba2..664c7e2553 100644
--- a/rules/config
+++ b/rules/config
@@ -224,6 +224,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
 #   docker: docker base images
 SONIC_VERSION_CONTROL_COMPONENTS ?= none
 
+# MIRROR_SNAPSHOT - support mirror snapshot flag
+MIRROR_SNAPSHOT ?= n
+
 # SONiC docker registry
 #
 # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry
diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh
index 5f94e701cb..a1e5900a48 100755
--- a/scripts/build_mirror_config.sh
+++ b/scripts/build_mirror_config.sh
@@ -5,16 +5,40 @@ CONFIG_PATH=$1
 export ARCHITECTURE=$2
 export DISTRIBUTION=$3
 
+DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
+[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror
+
 # The default mirror urls
 DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
 DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
 
+
 # The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
 if [ "$ARCHITECTURE" == "armhf" ]; then
     DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
     DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
 fi
 
+if [ "$MIRROR_SNAPSHOT" == y ]; then
+    if [ -f $MIRROR_VERSION_FILE ]; then
+        DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
+        DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
+    elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+    fi
+
+    DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/
+    DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/
+
+    mkdir -p target/versions/default
+    if [ ! -f target/versions/default/versions-mirror ]; then
+        echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+        echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+    fi
+fi
+
 [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
 [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
 
@@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2
 [ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
 
 MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
+if [ "$MIRROR_SNAPSHOT" == y ]; then
+    # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
+    sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
+fi
diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh
index fe7657a6b6..080e6a321a 100755
--- a/scripts/generate_buildinfo_config.sh
+++ b/scripts/generate_buildinfo_config.sh
@@ -8,3 +8,4 @@ mkdir -p $BUILDINFO_PATH/buildinfo/config
 
 echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
 echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG
diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py
index a20684e97b..d4127dece9 100755
--- a/scripts/versions_manager.py
+++ b/scripts/versions_manager.py
@@ -475,12 +475,13 @@ class VersionBuild:
             module.filter(ctypes=ctypes)
 
     def get_default_module(self):
-        if DEFAULT_MODULE in self.modules:
-            return self.modules[DEFAULT_MODULE]
+        default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, []))
         ctypes = self.get_component_types()
         dists = self.get_dists()
         components = []
         for ctype in ctypes:
+            if ctype in DEFAULT_OVERWRITE_COMPONENTS:
+                continue
             if ctype == 'deb':
                 for dist in dists:
                     versions = self._get_versions(ctype, dist)
@@ -492,7 +493,9 @@ class VersionBuild:
                 common_versions = self._get_common_versions(versions)
                 component = Component(common_versions, ctype)
                 components.append(component)
-        return VersionModule(DEFAULT_MODULE, components)
+        module = VersionModule(DEFAULT_MODULE, components)
+        module.overwrite(default_module, True, True)
+        return module
 
     def get_aggregatable_modules(self):
         modules = {}
diff --git a/slave.mk b/slave.mk
index 9173e73d9f..308182bba4 100644
--- a/slave.mk
+++ b/slave.mk
@@ -81,6 +81,7 @@ export IMAGE_DISTRO_DEBS_PATH
 export MULTIARCH_QEMU_ENVIRON
 export DOCKER_BASE_ARCH
 export BLDENV
+export MIRROR_SNAPSHOT
 
 ###############################################################################
 ## Utility rules
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
index ff249dfdfb..b9cbb5b3fb 100755
--- a/src/sonic-build-hooks/scripts/buildinfo_base.sh
+++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions
 VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb
 WEB_VERSION_FILE=$VERSION_PATH/versions-web
 BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
-REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
+REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
 DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
 
 . $BUILDINFO_PATH/config/buildinfo.config
@@ -72,14 +72,39 @@ set_reproducible_mirrors()
 {
     # Remove the charater # in front of the line if matched
     local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/"
+    # Add the character # in front of the line, if not match the URL pattern condition
+    local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/"
+    local expression3="\$a#SET_REPR_MIRRORS"
     if [ "$1" = "-d" ]; then
         # Add the charater # in front of the line if match
         expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/"
+        # Remove the character # in front of the line, if not match the URL pattern condition
+        expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/"
+        expression3="/#SET_REPR_MIRRORS/d"
     fi
 
     local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)"
     for mirror in $mirrors; do
+        if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then
+            continue
+        fi
+
+        # Make sure no duplicate operations on the mirror config file
+        if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") ||
+           ([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then
+            continue
+        fi
+
+        # Enable or disable the reproducible mirrors
         $SUDO sed -i "$expression" "$mirror"
+
+        # Enable or disable the none reproducible mirrors
+        if [ "$MIRROR_SNAPSHOT" == y ]; then
+            $SUDO sed -ri "$expression2" "$mirror"
+        fi
+
+        # Add or remove the SET_REPR_MIRRORS flag
+        $SUDO sed -i "$expression3" "$mirror"
     done
 }
 
diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files
index a4b33eeaa8..3fcc99a871 100755
--- a/src/sonic-build-hooks/scripts/collect_version_files
+++ b/src/sonic-build-hooks/scripts/collect_version_files
@@ -18,6 +18,13 @@ dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DI
 ## Add the the packages purged
 [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
 
+## Add mirror versions
+while read -r line; do
+    mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//")
+    date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ)
+    echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror
+done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null)
+
 ## Print the unique and sorted result
 sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
 if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then
@@ -26,5 +33,8 @@ fi
 if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then
     sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
 fi
+if [ -e "${TARGET_PATH}/versions-mirror" ]; then
+    sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror"
+fi
 
 exit 0