sonic-buildimage/scripts/prepare_docker_buildinfo.sh
Kalimuthu-Velappan 0dc22bd27c
05.Version cache - docker dpkg caching support (#12005)
This feature caches all the deb files during docker build and stores them
into version cache.

It loads the cache file if already exists in the version cache and copies the extracted
deb file from cache file into Debian cache path( /var/cache/apt/archives).

The apt-install always installs the deb file from the cache if exists, this
avoid unnecessary package download from the repo and speeds up the overall build.

The cache file is selected based on the SHA value of version dependency
files.

Why I did it
How I did it
How to verify it


* 03.Version-cache - framework environment settings

It defines and passes the necessary version cache environment variables
to the caching framework.

It adds the utils script for shared cache file access.

It also adds the post-cleanup logic for cleaning the unwanted files from
the docker/image after the version cache creation.

* 04.Version cache - debug framework

Added DBGOPT Make variable to enable the cache framework
scripts in trace mode. This option takes the part name of the script to
enable the particular shell script in trace mode.

Multiple shell script names can also be given.

	Eg: make DBGOPT="image|docker"

Added verbose mode to dump the version merge details during
build/dry-run mode.
	Eg: scripts/versions_manager.py freeze -v \
		'dryrun|cmod=docker-swss|cfile=versions-deb|cname=all|stage=sub|stage=add'

* 05.Version cache - docker dpkg caching support

This feature caches all the deb files during docker build and stores them
into version cache.

It loads the cache file if already exists in the version cache and copies the extracted
deb file from cache file into Debian cache path( /var/cache/apt/archives).

The apt-install always installs the deb file from the cache if exists, this
avoid unnecessary package download from the repo and speeds up the overall build.

The cache file is selected based on the SHA value of version dependency
files.
2022-12-12 09:20:56 +08:00

159 lines
5.4 KiB
Bash
Executable File

#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
BUILDINFO_BASE=/usr/local/share/buildinfo
SCRIPT_SRC_PATH=src/sonic-build-hooks
if [ -e ${SCRIPT_SRC_PATH} ]; then
. ${SCRIPT_SRC_PATH}/scripts/utils.sh
else
. ${BUILDINFO_BASE}/scripts/utils.sh
fi
IMAGENAME=$1
DOCKERFILE=$2
ARCH=$3
DOCKERFILE_TARGET=$4
DISTRO=$5
[ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n
[ -z "$DOCKERFILE_TARGET" ] && DOCKERFILE_TARGET=$DOCKERFILE
DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGET")
BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
DOCKER_PATH=$(dirname $DOCKERFILE)
[ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH
mkdir -p $BUILDINFO_VERSION_PATH
# Get the debian distribution from the docker base image
if [ -z "$DISTRO" ]; then
DOCKER_BASE_IMAGE=$(grep "^FROM" $DOCKERFILE | head -n 1 | awk '{print $2}')
DISTRO=$(docker run --rm --entrypoint "" $DOCKER_BASE_IMAGE cat /etc/os-release | grep VERSION_CODENAME | cut -d= -f2)
if [ -z "$DISTRO" ]; then
DISTRO=$(docker run --rm --entrypoint "" $DOCKER_BASE_IMAGE cat /etc/apt/sources.list | grep deb.debian.org | awk '{print $3}')
[ -z "$DISTRO" ] && DISTRO=jessie
fi
fi
if [[ "$IMAGENAME" == docker-base-* ]]; then
scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
fi
# add script for reproducible build. using sha256 instead of tag for docker base image.
scripts/docker_version_control.sh $@
DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
ARG SONIC_VERSION_CACHE
ARG SONIC_VERSION_CONTROL_COMPONENTS
COPY ["buildinfo", "/usr/local/share/buildinfo"]
COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
ENV IMAGENAME='${IMAGENAME}'
ENV DISTRO='${DISTRO}'
RUN pre_run_buildinfo '${IMAGENAME}'
'
# Add the auto-generate code if it is not added in the target Dockerfile
if [ ! -f $DOCKERFILE_TARGET ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGET; then
# Insert the docker build script before the RUN command
LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1)
TEMP_FILE=$(mktemp)
awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE
# Append the docker build script at the end of the docker file
echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
echo -e "\nRUN post_run_cleanup ${IMAGENAME} " >> $TEMP_FILE
cat $TEMP_FILE > $DOCKERFILE_TARGET
rm -f $TEMP_FILE
fi
# Copy the build info config
mkdir -p ${BUILDINFO_PATH}
cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
# Generate the version lock files
scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
touch $BUILDINFO_VERSION_PATH/versions-deb
# Create the cache directories
LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
if [[ "$SKIP_BUILD_HOOK" == y || ${ENABLE_VERSION_CONTROL_DOCKER} != y ]]; then
exit 0
fi
# Version cache
DOCKER_IMAGE_NAME=${IMAGENAME}
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
else
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
fi
SRC_VERSION_PATH=files/build/versions
if [ ! -z ${SONIC_VERSION_CACHE} ]; then
# Version files for SHA calculation
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
DEP_FILES="Dockerfile.j2"
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
DEP_DBG_FILES="build_debug_docker_j2.sh"
fi
#Calculate the version SHA
VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
(cd ${DOCKER_PATH}; cat ${DEP_FILES}); \
cat ${DEP_DBG_FILES} ${VERSION_FILES}) \
| sha1sum | awk '{print substr($1,0,23);}')"
GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
LOCAL_CACHE_FILE=${LOCAL_CACHE_DIR}/cache.tgz
GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
# Create the empty cache tar file as local cache
if [[ ! -f ${LOCAL_CACHE_FILE} ]]; then
tar -zcf ${LOCAL_CACHE_FILE} -T /dev/null
chmod -f 777 ${LOCAL_CACHE_FILE}
fi
# Global cache file exists, load from global cache.
if [[ -e ${GLOBAL_CACHE_FILE} ]]; then
cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
touch ${GLOBAL_CACHE_FILE}
else
# When file is modified, Global SHA is calculated with the local change.
# Load from the previous version of build cache if exists
VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )
for VERSION in ${VERSIONS[@]}; do
VERSION_PREV_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
(cd ${DOCKER_PATH}; git --no-pager show $(ls -f ${DEP_FILES}|sed 's|.*|'${VERSION}':./&|g')); \
(git --no-pager show $(ls -f ${DEP_DBG_FILES} ${VERSION_FILES}|sed 's|.*|'${VERSION}':&|g'))) \
| sha1sum | awk '{print substr($1,0,23);}')"
GLOBAL_PREV_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_PREV_SHA}.tgz
if [[ -e ${GLOBAL_PREV_CACHE_FILE} ]]; then
cp ${GLOBAL_PREV_CACHE_FILE} ${LOCAL_CACHE_FILE}
touch ${GLOBAL_PREV_CACHE_FILE}
break
fi
done
fi
rm -f ${DOCKER_PATH}/vcache/cache.tgz
ln -f ${LOCAL_CACHE_FILE} ${DOCKER_PATH}/vcache/cache.tgz
else
# Delete the cache file if version cache is disabled.
rm -f ${DOCKER_PATH}/vcache/cache.tgz
fi