From ac5d89c6acebc9cfd75e21ec411e7ef175aef069 Mon Sep 17 00:00:00 2001 From: xumia <59720581+xumia@users.noreply.github.com> Date: Wed, 9 Nov 2022 08:09:53 +0800 Subject: [PATCH] [Build] Support j2 template for debian sources (#12557) Why I did it Unify the Debian mirror sources Make easy to upgrade to the next Debian release, not source url code change required. Support to customize the Debian mirror sources during the build Relative issue: #12523 --- .gitignore | 3 ++ Makefile.work | 7 ++++ build_debian.sh | 1 + dockers/docker-base-bullseye/Dockerfile.j2 | 8 +--- dockers/docker-base-bullseye/sources.list | 14 ------- .../docker-base-bullseye/sources.list.arm64 | 9 ----- .../docker-base-bullseye/sources.list.armhf | 9 ----- dockers/docker-base-buster/Dockerfile.j2 | 8 +--- dockers/docker-base-buster/sources.list | 14 ------- dockers/docker-base-buster/sources.list.arm64 | 12 ------ dockers/docker-base-buster/sources.list.armhf | 12 ------ dockers/docker-base-stretch/Dockerfile.j2 | 8 +--- dockers/docker-base-stretch/sources.list | 11 ------ .../docker-base-stretch/sources.list.arm64 | 11 ------ .../docker-base-stretch/sources.list.armhf | 11 ------ files/apt/sources.list.amd64 | 13 ------- files/apt/sources.list.arm64 | 13 ------- files/apt/sources.list.armhf | 18 --------- files/apt/sources.list.j2 | 20 ++++++++++ scripts/build_mirror_config.sh | 26 +++++++++++++ scripts/prepare_docker_buildinfo.sh | 4 ++ slave.mk | 2 + sonic-slave-bullseye/Dockerfile.j2 | 37 ++----------------- sonic-slave-buster/Dockerfile.j2 | 36 ++---------------- sonic-slave-stretch/Dockerfile.j2 | 35 +----------------- 25 files changed, 75 insertions(+), 267 deletions(-) delete mode 100644 dockers/docker-base-bullseye/sources.list delete mode 100644 dockers/docker-base-bullseye/sources.list.arm64 delete mode 100644 dockers/docker-base-bullseye/sources.list.armhf delete mode 100644 dockers/docker-base-buster/sources.list delete mode 100644 dockers/docker-base-buster/sources.list.arm64 delete mode 100644 dockers/docker-base-buster/sources.list.armhf delete mode 100644 dockers/docker-base-stretch/sources.list delete mode 100644 dockers/docker-base-stretch/sources.list.arm64 delete mode 100644 dockers/docker-base-stretch/sources.list.armhf delete mode 100644 files/apt/sources.list.amd64 delete mode 100644 files/apt/sources.list.arm64 delete mode 100644 files/apt/sources.list.armhf create mode 100644 files/apt/sources.list.j2 create mode 100755 scripts/build_mirror_config.sh diff --git a/.gitignore b/.gitignore index 5f6f41b1a0..60c328d622 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,6 @@ htmlcov/ .vscode/ .idea/ +# Debian mirror Sources +sources.list.* +!sources.list*.j2 diff --git a/Makefile.work b/Makefile.work index 56f0cde6c3..b048282d64 100644 --- a/Makefile.work +++ b/Makefile.work @@ -494,6 +494,8 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \ ENABLE_FIPS=$(ENABLE_FIPS) \ SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \ + MIRROR_URLS=$(MIRROR_URLS) \ + MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset @@ -514,6 +516,9 @@ endif .DEFAULT_GOAL := all .SHELLFLAGS += -e +export MIRROR_URLS +export MIRROR_SECURITY_URLS + %:: | sonic-build-hooks ifneq ($(filter y, $(MULTIARCH_QEMU_ENVIRON) $(CROSS_BUILD_ENVIRON)),) $(Q)$(DOCKER_MULTIARCH_CHECK) @@ -539,6 +544,8 @@ sonic-build-hooks: $(Q)pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) $(MAKE) all; popd $(Q)mkdir -p $(SLAVE_DIR)/buildinfo $(Q)cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo + $(Q)[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV) + $(Q)scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV) sonic-slave-base-build : | sonic-build-hooks ifeq ($(MULTIARCH_QEMU_ENVIRON), y) diff --git a/build_debian.sh b/build_debian.sh index dd98b052e1..e09e23706f 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -108,6 +108,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount [ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/ ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates +scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ diff --git a/dockers/docker-base-bullseye/Dockerfile.j2 b/dockers/docker-base-bullseye/Dockerfile.j2 index 8d197d3c90..c237015500 100644 --- a/dockers/docker-base-bullseye/Dockerfile.j2 +++ b/dockers/docker-base-bullseye/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-bullseye/sources.list b/dockers/docker-base-bullseye/sources.list deleted file mode 100644 index c45ef0811f..0000000000 --- a/dockers/docker-base-bullseye/sources.list +++ /dev/null @@ -1,14 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian bullseye-updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free - -# Debian mirror supports multiple versions for a package -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/dockers/docker-base-bullseye/sources.list.arm64 b/dockers/docker-base-bullseye/sources.list.arm64 deleted file mode 100644 index 5a0652aa3c..0000000000 --- a/dockers/docker-base-bullseye/sources.list.arm64 +++ /dev/null @@ -1,9 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free -deb [arch=arm64] http://security.debian.org bullseye-security main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian bullseye-backports main -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free diff --git a/dockers/docker-base-bullseye/sources.list.armhf b/dockers/docker-base-bullseye/sources.list.armhf deleted file mode 100644 index 435f4692de..0000000000 --- a/dockers/docker-base-bullseye/sources.list.armhf +++ /dev/null @@ -1,9 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free -deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-backports main -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index 9118287cac..6977b44d34 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list deleted file mode 100644 index 473c9eb22e..0000000000 --- a/dockers/docker-base-buster/sources.list +++ /dev/null @@ -1,14 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free - -# Debian mirror supports multiple versions for a package -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64 deleted file mode 100644 index 249efc17b6..0000000000 --- a/dockers/docker-base-buster/sources.list.arm64 +++ /dev/null @@ -1,12 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free -deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free -deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free -deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf deleted file mode 100644 index ff6d5787b2..0000000000 --- a/dockers/docker-base-buster/sources.list.armhf +++ /dev/null @@ -1,12 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free -deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free -deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2 index 5603dc5027..9e11c4ea35 100644 --- a/dockers/docker-base-stretch/Dockerfile.j2 +++ b/dockers/docker-base-stretch/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-stretch/sources.list b/dockers/docker-base-stretch/sources.list deleted file mode 100644 index 0c29b339bb..0000000000 --- a/dockers/docker-base-stretch/sources.list +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch-backports main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.arm64 b/dockers/docker-base-stretch/sources.list.arm64 deleted file mode 100644 index 520c465199..0000000000 --- a/dockers/docker-base-stretch/sources.list.arm64 +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free -deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free -deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free -deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian/ stretch-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.armhf b/dockers/docker-base-stretch/sources.list.armhf deleted file mode 100644 index 58077f3104..0000000000 --- a/dockers/docker-base-stretch/sources.list.armhf +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free -deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free -deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian/ stretch-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/files/apt/sources.list.amd64 b/files/apt/sources.list.amd64 deleted file mode 100644 index fb0a1e9949..0000000000 --- a/files/apt/sources.list.amd64 +++ /dev/null @@ -1,13 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.arm64 b/files/apt/sources.list.arm64 deleted file mode 100644 index 75bebe1a91..0000000000 --- a/files/apt/sources.list.arm64 +++ /dev/null @@ -1,13 +0,0 @@ -## Debian mirror for ARM -## Not the repo mirror site can change in future, and needs to be updated to be in sync - -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=arm64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.armhf b/files/apt/sources.list.armhf deleted file mode 100644 index 4f9d4d0a90..0000000000 --- a/files/apt/sources.list.armhf +++ /dev/null @@ -1,18 +0,0 @@ -## Debian mirror for ARM -## Not the repo mirror site can change in future, and needs to be updated to be in sync - -deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb-src [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-backports main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -#deb-src [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -#deb-src [arch=armhf] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.j2 b/files/apt/sources.list.j2 new file mode 100644 index 0000000000..57d53b919b --- /dev/null +++ b/files/apt/sources.list.j2 @@ -0,0 +1,20 @@ +# The configuration is generated by template +# Please add additional sources in /etc/apt/sources.list.d + +{% for mirror_url in MIRROR_URLS.split(',') %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free +{% endfor %} +{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %} +{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %} +{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free +{% else %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free +{% endif %} +{% endfor %} diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh new file mode 100755 index 0000000000..5f94e701cb --- /dev/null +++ b/scripts/build_mirror_config.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Generate the sources.list. in the config path +CONFIG_PATH=$1 +export ARCHITECTURE=$2 +export DISTRIBUTION=$3 + +# The default mirror urls +DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/ +DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ + +# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead +if [ "$ARCHITECTURE" == "armhf" ]; then + DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/ + DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ +fi + +[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS +[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS + +TEMPLATE=files/apt/sources.list.j2 +[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2 +[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2 +[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2 + +MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index d2b2c57185..3c1104eddc 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then [ -z "$DISTRO" ] && DISTRO=jessie fi +if [[ "$IMAGENAME" == docker-base-* ]]; then + scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO +fi + # add script for reproducible build. using sha256 instead of tag for docker base image. scripts/docker_version_control.sh $@ diff --git a/slave.mk b/slave.mk index c358e965fa..32ca3cb2bf 100644 --- a/slave.mk +++ b/slave.mk @@ -137,6 +137,8 @@ export TRUSTED_GPG_URLS export SONIC_VERSION_CONTROL_COMPONENTS DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY) export DEFAULT_CONTAINER_REGISTRY +export MIRROR_URLS +export MIRROR_SECURITY_URLS ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y) ENABLE_PFCWD_ON_START = y diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index 2969057871..b22639e831 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -23,39 +23,10 @@ MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] -## TODO: Re-add in any necessary mirror URLs here as they become available -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian bullseye-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" and CROSS_BUILD_ENVIRON != "y" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" and CROSS_BUILD_ENVIRON != "y" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CROSS_BUILD_ENVIRON == "y" %} +{%- if CROSS_BUILD_ENVIRON != "y" %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] +{%- else %} +COPY ["sources.list.amd64", "/etc/apt/sources.list"] {%- if CONFIGURED_ARCH == "armhf" %} ARG arch=armhf ARG gcc_arch=arm-linux-gnueabihf diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index f2a91a27d5..73e9febda1 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -22,38 +22,10 @@ MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" and CROSS_BUILD_ENVIRON != "y" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" and CROSS_BUILD_ENVIRON != "y" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CROSS_BUILD_ENVIRON == "y" %} +{%- if CROSS_BUILD_ENVIRON != "y" %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] +{%- else %} +COPY ["sources.list.amd64", "/etc/apt/sources.list"] {%- if CONFIGURED_ARCH == "armhf" %} ARG arch=armhf ARG gcc_arch=arm-linux-gnueabihf diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index e3815eab2d..1397df1c73 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -10,40 +10,7 @@ FROM {{ prefix }}debian:stretch MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] - -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.microsoft.com/debian/9/prod stretch main" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list -{%- endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive