sonic-buildimage/scripts/build_debian_base_system.sh

118 lines
4.4 KiB
Bash
Raw Normal View History

#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
05.Version cache - docker dpkg caching support (#12005) This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files. Why I did it How I did it How to verify it * 03.Version-cache - framework environment settings It defines and passes the necessary version cache environment variables to the caching framework. It adds the utils script for shared cache file access. It also adds the post-cleanup logic for cleaning the unwanted files from the docker/image after the version cache creation. * 04.Version cache - debug framework Added DBGOPT Make variable to enable the cache framework scripts in trace mode. This option takes the part name of the script to enable the particular shell script in trace mode. Multiple shell script names can also be given. Eg: make DBGOPT="image|docker" Added verbose mode to dump the version merge details during build/dry-run mode. Eg: scripts/versions_manager.py freeze -v \ 'dryrun|cmod=docker-swss|cfile=versions-deb|cname=all|stage=sub|stage=add' * 05.Version cache - docker dpkg caching support This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files.
2022-12-11 19:20:56 -06:00
CONFIGURED_ARCH=$1
IMAGE_DISTRO=$2
FILESYSTEM_ROOT=$3
http_proxy=$4
TARGET=$TARGET_PATH
[ -z "$TARGET" ] && TARGET=target
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
TARGET_BASEIMAGE_PATH=$TARGET/versions/host-base-image
mkdir -p $TARGET_BASEIMAGE_PATH
alias urlencode='python3 -c "import sys, urllib.parse as ul; print (ul.quote_plus(sys.argv[1]))"'
shopt -s expand_aliases
generate_version_file()
{
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg-query -W -f '\${Package}==\${Version}\n'" > $TARGET_BASEIMAGE_PATH/versions-deb-${IMAGE_DISTRO}-${CONFIGURED_ARCH}
}
MIRROR_URL=http://deb.debian.org/debian
if [ "$MIRROR_SNAPSHOT" == y ]; then
SNAPSHOT_TIMESTAMP=$(grep "^debian==" $TARGET/versions/default/versions-mirror | tail -n 1 | sed 's/.*==//')
MIRROR_URL=http://packages.trafficmanager.net/snapshot/debian/$SNAPSHOT_TIMESTAMP
fi
if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ] || [ ! -d files/build/versions/host-base-image ]; then
if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
Ported Marvell armhf build on amd64 host for debian buster to use cross-comp… (#8035) * Ported Marvell armhf build on x86 for debian buster to use cross-compilation instead of qemu emulation Current armhf Sonic build on amd64 host uses qemu emulation. Due to the nature of the emulation it takes a very long time, about 22-24 hours to complete the build. The change I did to reduce the building time by porting Sonic armhf build on amd64 host for Marvell platform for debian buster to use cross-compilation on arm64 host for armhf target. The overall Sonic armhf building time using cross-compilation reduced to about 6 hours. Signed-off-by: marvell <marvell@cpss-build3.marvell.com> * Fixed final Sonic image build with dockers inside * Update Dockerfile.j2 Fixed qemu-user-static:x86_64-aarch64-5.0.0-2 . * Update cross-build-arm-python-reqirements.sh Added support for both armhf and arm64 cross-build platform using $PY_PLAT environment variable. * Update Makefile Added TARGET=<cross-target> for armhf/arm64 cross-compilation. * Reviewer's @qiluo-msft requests done Signed-off-by: marvell <marvell@cpss-build3.marvell.com> * Added new radius/pam patch for arm64 support * Update slave.mk Added missing back tick. * Added libgtest-dev: libgmock-dev: to the buster Dockerfile.j2. Fixed arm perl version to be generic * Added missing armhf/arm64 entries in /etc/apt/sources.list * fix libc-bin core dump issue from xumia:fix-libc-bin-install-issue commit * Removed unnecessary 'apt-get update' from sonic-slave-buster/Dockerfile.j2 * Fixed saiarcot895 reviewer's requests * Fixed README and replaced 'sed/awk' with patches * Fixed ntp build to use openssl * Unuse sonic-slave-buster/cross-build-arm-python-reqirements.sh script (put all prebuilt python packages cross-compilation/install inside Dockerfile.j2). Fixed src/snmpd/Makefile to use -j1 in all cases * Clean armhf cross-compilation build fixes * Ported cross-compilation armhf build to bullseye * Additional change for bullseye * Set CROSS_BUILD_ENVIRON default value n * Removed python2 references * Fixes after merge with the upstream * Deleted unused sonic-slave-buster/cross-build-arm-python-reqirements.sh file * Fixed 2 @saiarcot895 requests * Fixed @saiarcot895 reviewer's requests * Removed use of prebuilt python wheels * Incorporated saiarcot895 CC/CXX and other simplification/generalization changes Signed-off-by: marvell <marvell@cpss-build3.marvell.com> * Fixed saiarcot895 reviewer's additional requests * src/libyang/patch/debian-packaging-files.patch * Removed --no-deps option when installing wheels. Removed unnecessary lazy_object_proxy arm python3 package instalation Co-authored-by: marvell <marvell@cpss-build3.marvell.com> Co-authored-by: marvell <marvell@cpss-build2.marvell.com>
2022-07-21 16:15:16 -05:00
if [[ $MULTIARCH_QEMU_ENVIRON == y || $CROSS_BUILD_ENVIRON == y ]]; then
# qemu arm bin executable for cross-building
sudo mkdir -p $FILESYSTEM_ROOT/usr/bin
sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true
fi
sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --foreign --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT "$MIRROR_URL"
sudo rm $FILESYSTEM_ROOT/proc -rf
sudo mkdir $FILESYSTEM_ROOT/proc
sudo mount -t proc proc $FILESYSTEM_ROOT/proc
sudo LANG=C chroot $FILESYSTEM_ROOT /debootstrap/debootstrap --second-stage
else
sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT "$MIRROR_URL"
fi
RET=$?
if [ $RET -ne 0 ]; then
exit $RET
fi
generate_version_file
exit $RET
fi
ARCH=$(dpkg --print-architecture)
DISTRO=$(grep CODENAME /etc/os-release | cut -d= -f2)
if [ "$ARCH" != "$CONFIGURED_ARCH" ] || [ "$DISTRO" != "$IMAGE_DISTRO" ]; then
"Not support to build different ARCH/DISTRO ${CONFIGURED_ARCH}:${IMAGE_DISTRO} in ${ARCH}:${DISTRO}."
exit 1
fi
# Generate the version files for the host base image
TEMP_DIR=$(mktemp -d)
./scripts/versions_manager.py generate -t $TEMP_DIR -n host-base-image -d $IMAGE_DISTRO -a $CONFIGURED_ARCH
PACKAGES=$(sed -E 's/=(=[^=]*)$/\1/' $TEMP_DIR/versions-deb)
if [ -z "$PACKAGES" ]; then
echo "Not found host-base-image packages, please check the version files in files/build/versions/host-base-image" 2>&1
exit 1
fi
rm -rf $TEMP_DIR
BASEIMAGE_TARBALLPATH=$TARGET/baseimage
BASEIMAGE_TARBALL=$(realpath -e $TARGET)/baseimage.tgz
rm -rf $BASEIMAGE_TARBALLPATH $BASEIMAGE_TARBALL
ARCHIEVES=$BASEIMAGE_TARBALLPATH/var/cache/apt/archives
APTLIST=$BASEIMAGE_TARBALLPATH/var/lib/apt/lists
TARGET_DEBOOTSTRAP=$BASEIMAGE_TARBALLPATH/debootstrap
APTDEBIAN="$APTLIST/deb.debian.org_debian_dists_${DISTRO}_main_binary-${CONFIGURED_ARCH}_Packages"
DEBPATHS=$TARGET_DEBOOTSTRAP/debpaths
DEBOOTSTRAP_BASE=$TARGET_DEBOOTSTRAP/base
DEBOOTSTRAP_REQUIRED=$TARGET_DEBOOTSTRAP/required
[ -d $BASEIMAGE_TARBALLPATH ] && rm -rf $BASEIMAGE_TARBALLPATH
mkdir -p $ARCHIEVES
mkdir -p $APTLIST
mkdir -p $TARGET_DEBOOTSTRAP
URL_ARR=$(apt-get download --print-uris $PACKAGES | cut -d" " -f1 | tr -d "'")
PACKAGE_ARR=($PACKAGES)
LENGTH=${#PACKAGE_ARR[@]}
for ((i=0;i<LENGTH;i++))
do
package=${PACKAGE_ARR[$i]}
packagename=$(echo $package | sed -E 's/=[^=]*$//')
encoded_packagename=$(urlencode $packagename)
url=$(echo "$URL_ARR" | grep -i "/${packagename}_\|/${encoded_packagename}_")
if [ -z "$url" ] || [[ $(echo "$url" | wc -l) -gt 1 ]]; then
echo "No found package or found multiple package for package $packagename, url: $url" 1>&2
exit 1
fi
filename=$(basename "$url")
SKIP_BUILD_HOOK=y wget $url -O $ARCHIEVES/$filename
echo $packagename >> $DEBOOTSTRAP_REQUIRED
echo "$packagename /var/cache/apt/archives/$filename" >> $DEBPATHS
done
touch $APTDEBIAN
touch $DEBOOTSTRAP_BASE
(cd $BASEIMAGE_TARBALLPATH && fakeroot tar -zcf $BASEIMAGE_TARBALL .)
sudo debootstrap --verbose --variant=minbase --arch $CONFIGURED_ARCH --unpack-tarball=$BASEIMAGE_TARBALL $IMAGE_DISTRO $FILESYSTEM_ROOT
RET=$?
if [ $RET -ne 0 ]; then
exit $RET
fi
generate_version_file