matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

02.Version cache - docker cache build framework (#12001)

Browse Source 
During docker build, host files can be passed to the docker build through
docker context files. But there is no straightforward way to transfer
the files from docker build to host.

This feature provides a tricky way to pass the cache contents from docker
build to host. It tar's the cached content and encodes them as base64 format
and passes it through a log file with a special tag as 'VCSTART and VCENT'.

Slave.mk in the host, it extracts the cache contents from the log and stores them
in the cache folder. Cache contents are encoded as base64 format for
easy passing.

<!--
     Please make sure you've read and understood our contributing guidelines:
     https://github.com/Azure/SONiC/blob/gh-pages/CONTRIBUTING.md

     ** Make sure all your commits include a signature generated with `git commit -s` **

     If this is a bug fix, make sure your description includes "fixes #xxxx", or
     "closes #xxxx" or "resolves #xxxx"

     Please provide the following information:
-->

#### Why I did it

#### How I did it

#### How to verify it
This commit is contained in:
Kalimuthu-Velappan 2022-12-02 05:58:45 +05:30 committed by GitHub
parent 7a0152ad15
commit aaeafa8411
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 82 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile.work
View File

@ -228,8 +228,12 @@ SLAVE_TAG = $(shell \
COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
scripts/collect_docker_version_files.sh \
$(SLAVE_BASE_IMAGE) \
target \
$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
target
$(SLAVE_DIR) \
$(SLAVE_DIR)/Dockerfile
OVERLAY_MODULE_CHECK := \
lsmod | grep -q "^overlay " &>/dev/null || \
zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
@ -376,7 +380,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
--build-arg http_proxy=$(http_proxy) \
--build-arg https_proxy=$(https_proxy) \
--build-arg no_proxy=$(no_proxy) \
$(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_BASE_LOG)
$(SLAVE_DIR) \
$(SPLIT_LOG) $(DOCKER_BASE_LOG)
DOCKER_BASE_PULL = docker pull \
$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
@ -507,7 +512,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \
ifeq ($(filter clean,$(MAKECMDGOALS)),)
COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh \$$?; }
COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; }
endif
ifdef SOURCE_FOLDER

11
files/build_templates/build_docker_cache.j2 Normal file
View File

@ -0,0 +1,11 @@
# Base docker build
FROM {{IMAGENAME}}
# Copy the cache data to host
From scratch as output
COPY --from={{IMAGENAME}} /cache.tgz cache.tgz
# Clean up the cache data
FROM {{IMAGENAME}} as final
RUN rm /cache.tgz

2
scripts/collect_build_version_files.sh
View File

@ -23,6 +23,6 @@ mkdir -p $VERSION_SLAVE_PATH
scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
rm -rf $BUILD_VERSION_PATH/*
[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*
exit $RET

17
scripts/collect_docker_version_files.sh
View File

@ -1,12 +1,17 @@
#!/bin/bash
set -x
DOCKER_IMAGE=$1
TARGET_PATH=$2
DOCKER_IMAGE_TAG=$3
DOCKER_PATH=$4
DOCKER_FILE=$5
[ -z "$TARGET_PATH" ] && TARGET_PATH=./target
DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\$//")
DOCKER_CONTAINER=$DOCKER_IMAGE_NAME
#Create the container specific to the user tag and slave tag
DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
[ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
@ -18,8 +23,16 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then
docker container rm $DOCKER_IMAGE > /dev/null
fi
docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE
docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/
# Save the cache contents from docker build
IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
docker rmi tmp-${DOCKER_IMAGE_TAG}
docker container rm $DOCKER_CONTAINER

4
scripts/collect_host_image_version_files.sh
View File

@ -3,11 +3,13 @@
TARGET=$1
FILESYSTEM_ROOT=$2
VERSIONS_PATH=$TARGET/versions/host-image
IMAGENAME="host-image"
[ -d $VERSIONS_PATH ] && sudo rm -rf $VERSIONS_PATH
mkdir -p $VERSIONS_PATH
sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo
mkdir -p target/vcache/${IMAGENAME}
sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/

13
scripts/prepare_docker_buildinfo.sh
View File

@ -11,6 +11,7 @@ DISTRO=$5
DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
DOCKER_PATH=$(dirname $DOCKERFILE)
[ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH
mkdir -p $BUILDINFO_VERSION_PATH
@ -31,8 +32,11 @@ scripts/docker_version_control.sh $@
DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
COPY ["buildinfo", "/usr/local/share/buildinfo"]
COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
RUN pre_run_buildinfo'
ENV IMAGENAME='${IMAGENAME}'
RUN pre_run_buildinfo '${IMAGENAME}'
'
# Add the auto-generate code if it is not added in the target Dockerfile
if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
@ -42,7 +46,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE
awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE
# Append the docker build script at the end of the docker file
echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE
echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
cat $TEMP_FILE > $DOCKERFILE_TARGE
rm -f $TEMP_FILE
@ -55,3 +59,8 @@ cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
touch $BUILDINFO_VERSION_PATH/versions-deb
# Create the cache directories
LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/

15
slave.mk
View File

@ -111,6 +111,7 @@ configure :
$(Q)mkdir -p $(PYTHON_DEBS_PATH)
$(Q)mkdir -p $(PYTHON_WHEELS_PATH)
$(Q)mkdir -p $(DPKG_ADMINDIR_PATH)
$(Q)mkdir -p $(TARGET_PATH)/vcache
$(Q)echo $(PLATFORM) > .platform
$(Q)echo $(PLATFORM_ARCH) > .arch
@ -914,8 +915,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
--label Tag=$(SONIC_IMAGE_VERSION) \
-f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@ -993,6 +996,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
mkdir -p $($*.gz_PATH)/files $(LOG)
mkdir -p $($*.gz_PATH)/python-debs $(LOG)
mkdir -p $($*.gz_PATH)/python-wheels $(LOG)
mkdir -p $(TARGET_PATH)/vcache/$* $($*.gz_PATH)/vcache $(LOG)
sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
sudo mount --bind $($*.gz_FILES_PATH) $($*.gz_PATH)/files $(LOG)
sudo mount --bind $(PYTHON_DEBS_PATH) $($*.gz_PATH)/python-debs $(LOG)
@ -1034,9 +1038,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
--label Tag=$(SONIC_IMAGE_VERSION) \
$($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@ -1065,6 +1071,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
mkdir -p $($*.gz_PATH)/debs $(LOG)
sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
mkdir -p $(TARGET_PATH)/vcache/$*-dbg $($*.gz_PATH)/vcache $(LOG)
# Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs
$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++'))
@ -1088,9 +1095,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
--label Tag=$(SONIC_IMAGE_VERSION) \
--file $($*.gz_PATH)/Dockerfile-dbg \
-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
# Clean up
docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true

2
sonic-slave-buster/Dockerfile.j2
View File

@ -599,7 +599,7 @@ RUN add-apt-repository \
$(lsb_release -cs) \
stable"
RUN apt-get update
RUN apt-get install -y docker-ce=5:18.09.5~3-0~debian-buster docker-ce-cli=5:18.09.5~3-0~debian-buster
RUN apt-get install -y docker-ce=5:20.10.21~3-0~debian-buster docker-ce-cli=5:20.10.21~3-0~debian-buster
RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs {{ DOCKER_EXTRA_OPTS }}\"" >> /etc/default/docker
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy

9
src/sonic-build-hooks/scripts/buildinfo_base.sh
View File

@ -15,6 +15,15 @@ REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
. $BUILDINFO_PATH/config/buildinfo.config
if [ -e /vcache ]; then
PKG_CACHE_PATH=/vcache/${IMAGENAME}
else
PKG_CACHE_PATH=/sonic/target/vcache/${IMAGENAME}
fi
PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz
mkdir -p ${PKG_CACHE_PATH}
URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#")

6
src/sonic-build-hooks/scripts/post_run_buildinfo
View File

@ -1,12 +1,14 @@
#!/bin/bash
IMAGENAME=$1
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
# Collect the version files
collect_version_files $POST_VERSION_PATH
[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
rm -rf $BUILD_VERSION_PATH/*
#Save the cache file for exporting it to host.
tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
# Disable the build hooks
symlink_build_hooks -d

6
src/sonic-build-hooks/scripts/pre_run_buildinfo
View File

@ -1,5 +1,7 @@
#!/bin/bash
IMAGENAME=$1
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
[ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH
@ -15,10 +17,14 @@ update_version_files
symlink_build_hooks
set_reproducible_mirrors
mkdir -p /var/cache/apt/archives/
chmod -R a+rw $BUILDINFO_PATH
if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then
cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
fi
DISTRO=${DISTRO} apt-get update && apt-get install -y rsync
exit 0