0dc22bd27c
This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files. Why I did it How I did it How to verify it * 03.Version-cache - framework environment settings It defines and passes the necessary version cache environment variables to the caching framework. It adds the utils script for shared cache file access. It also adds the post-cleanup logic for cleaning the unwanted files from the docker/image after the version cache creation. * 04.Version cache - debug framework Added DBGOPT Make variable to enable the cache framework scripts in trace mode. This option takes the part name of the script to enable the particular shell script in trace mode. Multiple shell script names can also be given. Eg: make DBGOPT="image|docker" Added verbose mode to dump the version merge details during build/dry-run mode. Eg: scripts/versions_manager.py freeze -v \ 'dryrun|cmod=docker-swss|cfile=versions-deb|cname=all|stage=sub|stage=add' * 05.Version cache - docker dpkg caching support This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files.
112 lines
4.2 KiB
Bash
Executable File
112 lines
4.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
|
|
|
|
CONFIGURED_ARCH=$1
|
|
IMAGE_DISTRO=$2
|
|
FILESYSTEM_ROOT=$3
|
|
http_proxy=$4
|
|
|
|
TARGET=$TARGET_PATH
|
|
[ -z "$TARGET" ] && TARGET=target
|
|
|
|
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
|
|
|
|
TARGET_BASEIMAGE_PATH=$TARGET/versions/host-base-image
|
|
mkdir -p $TARGET_BASEIMAGE_PATH
|
|
|
|
alias urlencode='python3 -c "import sys, urllib.parse as ul; print (ul.quote_plus(sys.argv[1]))"'
|
|
shopt -s expand_aliases
|
|
|
|
generate_version_file()
|
|
{
|
|
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg-query -W -f '\${Package}==\${Version}\n'" > $TARGET_BASEIMAGE_PATH/versions-deb-${IMAGE_DISTRO}-${CONFIGURED_ARCH}
|
|
}
|
|
|
|
if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ] || [ ! -d files/build/versions/host-base-image ]; then
|
|
if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
|
|
if [[ $MULTIARCH_QEMU_ENVIRON == y || $CROSS_BUILD_ENVIRON == y ]]; then
|
|
# qemu arm bin executable for cross-building
|
|
sudo mkdir -p $FILESYSTEM_ROOT/usr/bin
|
|
sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true
|
|
fi
|
|
sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --foreign --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian
|
|
sudo rm $FILESYSTEM_ROOT/proc -rf
|
|
sudo mkdir $FILESYSTEM_ROOT/proc
|
|
sudo mount -t proc proc $FILESYSTEM_ROOT/proc
|
|
sudo LANG=C chroot $FILESYSTEM_ROOT /debootstrap/debootstrap --second-stage
|
|
else
|
|
sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian
|
|
fi
|
|
RET=$?
|
|
if [ $RET -ne 0 ]; then
|
|
exit $RET
|
|
fi
|
|
|
|
generate_version_file
|
|
exit $RET
|
|
fi
|
|
|
|
ARCH=$(dpkg --print-architecture)
|
|
DISTRO=$(grep CODENAME /etc/os-release | cut -d= -f2)
|
|
if [ "$ARCH" != "$CONFIGURED_ARCH" ] || [ "$DISTRO" != "$IMAGE_DISTRO" ]; then
|
|
"Not support to build different ARCH/DISTRO ${CONFIGURED_ARCH}:${IMAGE_DISTRO} in ${ARCH}:${DISTRO}."
|
|
exit 1
|
|
fi
|
|
|
|
# Generate the version files for the host base image
|
|
TEMP_DIR=$(mktemp -d)
|
|
./scripts/versions_manager.py generate -t $TEMP_DIR -n host-base-image -d $IMAGE_DISTRO -a $CONFIGURED_ARCH
|
|
PACKAGES=$(sed -E 's/=(=[^=]*)$/\1/' $TEMP_DIR/versions-deb)
|
|
if [ -z "$PACKAGES" ]; then
|
|
echo "Not found host-base-image packages, please check the version files in files/build/versions/host-base-image" 2>&1
|
|
exit 1
|
|
fi
|
|
rm -rf $TEMP_DIR
|
|
|
|
BASEIMAGE_TARBALLPATH=$TARGET/baseimage
|
|
BASEIMAGE_TARBALL=$(realpath -e $TARGET)/baseimage.tgz
|
|
|
|
rm -rf $BASEIMAGE_TARBALLPATH $BASEIMAGE_TARBALL
|
|
|
|
ARCHIEVES=$BASEIMAGE_TARBALLPATH/var/cache/apt/archives
|
|
APTLIST=$BASEIMAGE_TARBALLPATH/var/lib/apt/lists
|
|
TARGET_DEBOOTSTRAP=$BASEIMAGE_TARBALLPATH/debootstrap
|
|
APTDEBIAN="$APTLIST/deb.debian.org_debian_dists_${DISTRO}_main_binary-${CONFIGURED_ARCH}_Packages"
|
|
DEBPATHS=$TARGET_DEBOOTSTRAP/debpaths
|
|
DEBOOTSTRAP_BASE=$TARGET_DEBOOTSTRAP/base
|
|
DEBOOTSTRAP_REQUIRED=$TARGET_DEBOOTSTRAP/required
|
|
[ -d $BASEIMAGE_TARBALLPATH ] && rm -rf $BASEIMAGE_TARBALLPATH
|
|
mkdir -p $ARCHIEVES
|
|
mkdir -p $APTLIST
|
|
mkdir -p $TARGET_DEBOOTSTRAP
|
|
URL_ARR=$(apt-get download --print-uris $PACKAGES | cut -d" " -f1 | tr -d "'")
|
|
PACKAGE_ARR=($PACKAGES)
|
|
LENGTH=${#PACKAGE_ARR[@]}
|
|
for ((i=0;i<LENGTH;i++))
|
|
do
|
|
package=${PACKAGE_ARR[$i]}
|
|
packagename=$(echo $package | sed -E 's/=[^=]*$//')
|
|
encoded_packagename=$(urlencode $packagename)
|
|
url=$(echo "$URL_ARR" | grep -i "/${packagename}_\|/${encoded_packagename}_")
|
|
if [ -z "$url" ] || [[ $(echo "$url" | wc -l) -gt 1 ]]; then
|
|
echo "No found package or found multiple package for package $packagename, url: $url" 1>&2
|
|
exit 1
|
|
fi
|
|
filename=$(basename "$url")
|
|
SKIP_BUILD_HOOK=y wget $url -O $ARCHIEVES/$filename
|
|
echo $packagename >> $DEBOOTSTRAP_REQUIRED
|
|
echo "$packagename /var/cache/apt/archives/$filename" >> $DEBPATHS
|
|
done
|
|
touch $APTDEBIAN
|
|
touch $DEBOOTSTRAP_BASE
|
|
(cd $BASEIMAGE_TARBALLPATH && fakeroot tar -zcf $BASEIMAGE_TARBALL .)
|
|
|
|
sudo debootstrap --verbose --variant=minbase --arch $CONFIGURED_ARCH --unpack-tarball=$BASEIMAGE_TARBALL $IMAGE_DISTRO $FILESYSTEM_ROOT
|
|
RET=$?
|
|
if [ $RET -ne 0 ]; then
|
|
exit $RET
|
|
fi
|
|
|
|
generate_version_file
|