matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

05.Version cache - docker dpkg caching support (#12005)

Browse Source 
This feature caches all the deb files during docker build and stores them
into version cache.

It loads the cache file if already exists in the version cache and copies the extracted
deb file from cache file into Debian cache path( /var/cache/apt/archives).

The apt-install always installs the deb file from the cache if exists, this
avoid unnecessary package download from the repo and speeds up the overall build.

The cache file is selected based on the SHA value of version dependency
files.

Why I did it
How I did it
How to verify it


* 03.Version-cache - framework environment settings

It defines and passes the necessary version cache environment variables
to the caching framework.

It adds the utils script for shared cache file access.

It also adds the post-cleanup logic for cleaning the unwanted files from
the docker/image after the version cache creation.

* 04.Version cache - debug framework

Added DBGOPT Make variable to enable the cache framework
scripts in trace mode. This option takes the part name of the script to
enable the particular shell script in trace mode.

Multiple shell script names can also be given.

	Eg: make DBGOPT="image|docker"

Added verbose mode to dump the version merge details during
build/dry-run mode.
	Eg: scripts/versions_manager.py freeze -v \
		'dryrun|cmod=docker-swss|cfile=versions-deb|cname=all|stage=sub|stage=add'

* 05.Version cache - docker dpkg caching support

This feature caches all the deb files during docker build and stores them
into version cache.

It loads the cache file if already exists in the version cache and copies the extracted
deb file from cache file into Debian cache path( /var/cache/apt/archives).

The apt-install always installs the deb file from the cache if exists, this
avoid unnecessary package download from the repo and speeds up the overall build.

The cache file is selected based on the SHA value of version dependency
files.
This commit is contained in:
Kalimuthu-Velappan 2022-12-12 06:50:56 +05:30 committed by GitHub
parent 5624d15a7c
commit 0dc22bd27c
No account linked to committer's email address
21 changed files with 586 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Makefile.work
View File

@ -175,11 +175,22 @@ ifeq ($(ENABLE_FIPS), y)
endif
endif
SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
$(shell test -d $(SONIC_VERSION_CACHE_SOURCE) || \
mkdir -p $(SONIC_VERSION_CACHE_SOURCE) && chmod -f 777 $(SONIC_VERSION_CACHE_SOURCE) 2>/dev/null )
# Generate the version control build info
$(shell \
SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
DISTRO=$(BLDENV) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
DBGOPT='$(DBGOPT)' \
scripts/generate_buildinfo_config.sh)
# Generate the slave Dockerfile, and prepare build info for it
@ -198,6 +209,8 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
PREPARE_DOCKER=BUILD_SLAVE=y \
DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
scripts/prepare_docker_buildinfo.sh \
$(SLAVE_BASE_IMAGE) \
$(SLAVE_DIR)/Dockerfile \
@ -220,12 +233,13 @@ SLAVE_TAG = $(shell \
(cat $(SLAVE_DIR)/Dockerfile.user \
$(SLAVE_DIR)/Dockerfile \
$(SLAVE_DIR)/buildinfo/versions/versions-* \
.git/HEAD \
&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
| sha1sum \
| awk '{print substr($$1,0,11);}')
COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
scripts/collect_docker_version_files.sh \
$(SLAVE_BASE_IMAGE) \
target \
@ -286,6 +300,10 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
endif
ifneq ($(SONIC_VERSION_CACHE_SOURCE),)
DOCKER_RUN += -v "$(SONIC_VERSION_CACHE_SOURCE):/vcache:rw"
endif
ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y)
ifneq ($(SIGNING_KEY),)
DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY))
@ -379,6 +397,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
--build-arg http_proxy=$(http_proxy) \
--build-arg https_proxy=$(https_proxy) \
--build-arg no_proxy=$(no_proxy) \
--build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
--build-arg SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
$(SLAVE_DIR) \
$(SPLIT_LOG) $(DOCKER_BASE_LOG)

13
build_debian.sh
View File

@ -81,7 +81,10 @@ echo '[INFO] Build host debian base system...'
TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
# Prepare buildinfo
sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
sudo SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \
DBGOPT="${DBGOPT}" \
scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
sudo chown root:root $FILESYSTEM_ROOT
@ -440,10 +443,10 @@ if [[ $TARGET_BOOTLOADER == grub ]]; then
GRUB_PKG=grub-efi-arm64-bin
fi
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y download \
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -d -o dir::cache=/var/cache/apt \
$GRUB_PKG
sudo mv $FILESYSTEM_ROOT/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
sudo cp $FILESYSTEM_ROOT/var/cache/apt/archives/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
fi
## Disable kexec supported reboot which was installed by default
@ -635,7 +638,9 @@ if [[ $TARGET_BOOTLOADER == uboot ]]; then
fi
# Collect host image version files before cleanup
scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \
DBGOPT="${DBGOPT}" \
scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT
# Remove GCC
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc

5
platform/broadcom/sai.mk
View File

@ -17,4 +17,9 @@ $(BRCM_DNX_SAI)_URL = "$(LIBSAIBCM_DNX_URL_PREFIX)/$(BRCM_DNX_SAI)"
SONIC_ONLINE_DEBS += $(BRCM_XGS_SAI)
SONIC_ONLINE_DEBS += $(BRCM_DNX_SAI)
$(BRCM_XGS_SAI_DEV)_DEPENDS += $(BRCM_XGS_SAI)
$(BRCM_XGS_SAI)_SKIP_VERSION=y
$(BRCM_XGS_SAI_DEV)_SKIP_VERSION=y
$(BRCM_DNX_SAI)_SKIP_VERSION=y
$(eval $(call add_conflict_package,$(BRCM_XGS_SAI_DEV),$(LIBSAIVS_DEV)))

9
rules/config
View File

@ -229,6 +229,15 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
# docker: docker base images
SONIC_VERSION_CONTROL_COMPONENTS ?= none
# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
# cache standard installation package and restored back to avoid the package download every time.
# SONIC_VERSION_CACHE - Method of deb package caching
# none : no caching
# cache : Use cache if exists else build the source and update the cache
# SONIC_VERSION_CACHE_SOURCE - Defines the version cache location details
SONIC_VERSION_CACHE_METHOD ?= none
SONIC_VERSION_CACHE_SOURCE ?= $(SONIC_DPKG_CACHE_SOURCE)/vcache
# SONiC docker registry
#
# Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry

2
scripts/build_debian_base_system.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
CONFIGURED_ARCH=$1
IMAGE_DISTRO=$2
FILESYSTEM_ROOT=$3

7
scripts/collect_build_version_files.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
RET=$1
BLDENV=$2
TARGET_PATH=$3
@ -15,12 +17,17 @@ TIMESTAMP=$(date +"%Y%m%d%H%M%S")
VERSION_BUILD_PATH=$TARGET_PATH/versions/build
VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV}
LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP}
DEFAULT_VERSION_PATH=files/build/versions/default
BUILD_LOG_PATH=/sonic/target/versions/log/sonic-slave-${BLDENV}/
sudo chmod -R a+rw $BUILDINFO_PATH
collect_version_files $LOG_VERSION_PATH
([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/
mkdir -p $VERSION_SLAVE_PATH
mkdir -p ${BUILD_LOG_PATH}
([ -d ${LOG_PATH} ] && [ ! -z "$(ls ${LOG_PATH})" ]) && cp ${LOG_PATH}/* ${BUILD_LOG_PATH}
scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*

66
scripts/collect_docker_version_files.sh
View File

@ -1,6 +1,16 @@
#!/bin/bash
set -x
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
BUILDINFO_BASE=/usr/local/share/buildinfo
SCRIPT_SRC_PATH=src/sonic-build-hooks
if [ -e ${SCRIPT_SRC_PATH} ]; then
. ${SCRIPT_SRC_PATH}/scripts/utils.sh
else
. ${BUILDINFO_BASE}/scripts/utils.sh
fi
DOCKER_IMAGE=$1
TARGET_PATH=$2
DOCKER_IMAGE_TAG=$3
@ -13,6 +23,8 @@ DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\
#Create the container specific to the user tag and slave tag
DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
BUILD_LOG_PATH=target/versions/log/$DOCKER_IMAGE_NAME
mkdir -p ${BUILD_LOG_PATH}
[ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
mkdir -p $TARGET_VERSIONS_PATH
@ -34,5 +46,57 @@ docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
docker rmi tmp-${DOCKER_IMAGE_TAG}
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
# Save the cache contents from docker build
LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
sleep 1; sync ${CACHE_ENCODE_FILE}
# Decode the cache content into gz format
SRC_VERSION_PATH=files/build/versions
if [[ -e ${CACHE_ENCODE_FILE} ]]; then
cat ${CACHE_ENCODE_FILE} | base64 -d >${LOCAL_CACHE_FILE}
rm -f ${CACHE_ENCODE_FILE}
fi
# Version package cache
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
else
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
fi
if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
# Select version files for SHA calculation
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
fi
# Calculate the version SHA
VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; cat ${DEP_FILES} ${VERSION_FILES}) | sha1sum | awk '{print substr($1,0,23);}')"
GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
# If the cache file is not exists in the global cache for the given SHA,
# store the new cache file into version cache path.
if [ -f ${LOCAL_CACHE_FILE} ]; then
if [[ -z ${GIT_FILE_STATUS} && ! -e ${GLOBAL_CACHE_FILE} ]]; then
mkdir -p ${GLOBAL_CACHE_DIR}
chmod -f 777 ${GLOBAL_CACHE_DIR}
FLOCK ${GLOBAL_CACHE_FILE}
cp ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE}
chmod -f 777 ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE}
FUNLOCK ${GLOBAL_CACHE_FILE}
fi
fi
fi
docker container rm $DOCKER_CONTAINER

15
scripts/collect_host_image_version_files.sh
View File

@ -1,7 +1,16 @@
#!/bin/bash
TARGET=$1
FILESYSTEM_ROOT=$2
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
SCRIPT_SRC_PATH=src/sonic-build-hooks
if [ -e ${SCRIPT_SRC_PATH} ]; then
. ${SCRIPT_SRC_PATH}/scripts/utils.sh
fi
ARCH=$1
DISTRO=$2
TARGET=$3
FILESYSTEM_ROOT=$4
VERSIONS_PATH=$TARGET/versions/host-image
IMAGENAME="host-image"
@ -13,3 +22,5 @@ sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
sudo LANG=C chroot $FILESYSTEM_ROOT post_run_cleanup ${IMAGENAME}

7
scripts/generate_buildinfo_config.sh
View File

@ -6,5 +6,8 @@ BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config
mkdir -p $BUILDINFO_PATH/buildinfo/config
echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
echo "export PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG

10
scripts/prepare_debian_image_buildinfo.sh
View File

@ -1,9 +1,13 @@
#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
ARCH=$1
DISTRO=$2
FILESYSTEM_ROOT=$3
HOST_IMAGE_NAME=host-image
IMAGENAME=${HOST_IMAGE_NAME}
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
VERSION_DEB_PREFERENCE="01-versions-deb"
@ -26,4 +30,8 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then
fi
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb"
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
#sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export DISTRO=${DISTRO} >> /usr/local/share/buildinfo/config/buildinfo.config"
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export IMAGENAME=${IMAGENAME} >> /usr/local/share/buildinfo/config/buildinfo.config"
sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo ${HOST_IMAGE_NAME}"

99
scripts/prepare_docker_buildinfo.sh
View File

@ -1,14 +1,26 @@
#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
BUILDINFO_BASE=/usr/local/share/buildinfo
SCRIPT_SRC_PATH=src/sonic-build-hooks
if [ -e ${SCRIPT_SRC_PATH} ]; then
. ${SCRIPT_SRC_PATH}/scripts/utils.sh
else
. ${BUILDINFO_BASE}/scripts/utils.sh
fi
IMAGENAME=$1
DOCKERFILE=$2
ARCH=$3
DOCKERFILE_TARGE=$4
DOCKERFILE_TARGET=$4
DISTRO=$5
[ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n
[ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE
DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
[ -z "$DOCKERFILE_TARGET" ] && DOCKERFILE_TARGET=$DOCKERFILE
DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGET")
BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
DOCKER_PATH=$(dirname $DOCKERFILE)
@ -34,15 +46,18 @@ fi
scripts/docker_version_control.sh $@
DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
ARG SONIC_VERSION_CACHE
ARG SONIC_VERSION_CONTROL_COMPONENTS
COPY ["buildinfo", "/usr/local/share/buildinfo"]
COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
ENV IMAGENAME='${IMAGENAME}'
ENV DISTRO='${DISTRO}'
RUN pre_run_buildinfo '${IMAGENAME}'
'
# Add the auto-generate code if it is not added in the target Dockerfile
if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
if [ ! -f $DOCKERFILE_TARGET ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGET; then
# Insert the docker build script before the RUN command
LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1)
TEMP_FILE=$(mktemp)
@ -50,12 +65,14 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE
# Append the docker build script at the end of the docker file
echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
echo -e "\nRUN post_run_cleanup ${IMAGENAME} " >> $TEMP_FILE
cat $TEMP_FILE > $DOCKERFILE_TARGE
cat $TEMP_FILE > $DOCKERFILE_TARGET
rm -f $TEMP_FILE
fi
# Copy the build info config
mkdir -p ${BUILDINFO_PATH}
cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
# Generate the version lock files
@ -67,3 +84,75 @@ touch $BUILDINFO_VERSION_PATH/versions-deb
LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
if [[ "$SKIP_BUILD_HOOK" == y || ${ENABLE_VERSION_CONTROL_DOCKER} != y ]]; then
exit 0
fi
# Version cache
DOCKER_IMAGE_NAME=${IMAGENAME}
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
else
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
fi
SRC_VERSION_PATH=files/build/versions
if [ ! -z ${SONIC_VERSION_CACHE} ]; then
# Version files for SHA calculation
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
DEP_FILES="Dockerfile.j2"
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
DEP_DBG_FILES="build_debug_docker_j2.sh"
fi
#Calculate the version SHA
VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
(cd ${DOCKER_PATH}; cat ${DEP_FILES}); \
cat ${DEP_DBG_FILES} ${VERSION_FILES}) \
| sha1sum | awk '{print substr($1,0,23);}')"
GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
LOCAL_CACHE_FILE=${LOCAL_CACHE_DIR}/cache.tgz
GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
# Create the empty cache tar file as local cache
if [[ ! -f ${LOCAL_CACHE_FILE} ]]; then
tar -zcf ${LOCAL_CACHE_FILE} -T /dev/null
chmod -f 777 ${LOCAL_CACHE_FILE}
fi
# Global cache file exists, load from global cache.
if [[ -e ${GLOBAL_CACHE_FILE} ]]; then
cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
touch ${GLOBAL_CACHE_FILE}
else
# When file is modified, Global SHA is calculated with the local change.
# Load from the previous version of build cache if exists
VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )
for VERSION in ${VERSIONS[@]}; do
VERSION_PREV_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
(cd ${DOCKER_PATH}; git --no-pager show $(ls -f ${DEP_FILES}|sed 's|.*|'${VERSION}':./&|g')); \
(git --no-pager show $(ls -f ${DEP_DBG_FILES} ${VERSION_FILES}|sed 's|.*|'${VERSION}':&|g'))) \
| sha1sum | awk '{print substr($1,0,23);}')"
GLOBAL_PREV_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_PREV_SHA}.tgz
if [[ -e ${GLOBAL_PREV_CACHE_FILE} ]]; then
cp ${GLOBAL_PREV_CACHE_FILE} ${LOCAL_CACHE_FILE}
touch ${GLOBAL_PREV_CACHE_FILE}
break
fi
done
fi
rm -f ${DOCKER_PATH}/vcache/cache.tgz
ln -f ${LOCAL_CACHE_FILE} ${DOCKER_PATH}/vcache/cache.tgz
else
# Delete the cache file if version cache is disabled.
rm -f ${DOCKER_PATH}/vcache/cache.tgz
fi

15
scripts/prepare_slave_container_buildinfo.sh
View File

@ -1,14 +1,19 @@
#!/bin/bash
[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x
SLAVE_DIR=$1
ARCH=$2
DISTRO=$3
# Install the latest debian package sonic-build-hooks in the slave container
sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb > /dev/null
sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb &> /dev/null
# Enable the build hooks
symlink_build_hooks
sudo symlink_build_hooks
# set the global permissions
sudo chmod -f 777 /usr/local/share/buildinfo/ -R
# Build the slave running config
cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/
@ -21,8 +26,8 @@ apt-get update > /dev/null 2>&1
# Build the slave version config
[ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions
scripts/versions_manager.py generate -t "/usr/local/share/buildinfo/versions" -n "build-${SLAVE_DIR}" -d "$DISTRO" -a "$ARCH"
touch ${BUILDINFO_PATH}/versions/versions-deb
touch ${BUILDINFO_PATH}/versions/versions-deb ${BUILDINFO_PATH}/versions/versions-web
rm -f /etc/apt/preferences.d/01-versions-deb
([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
sudo rm -f /etc/apt/preferences.d/01-versions-deb
([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && sudo cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
exit 0

156
scripts/versions_manager.py
View File

@ -4,6 +4,7 @@ import argparse
import glob
import os
import sys
import re
ALL_DIST = 'all'
ALL_ARCH = 'all'
@ -24,7 +25,7 @@ class Component:
arch -- Architectrue, such as amd64, arm64, etc
'''
def __init__(self, versions, ctype, dist=ALL_DIST, arch=ALL_ARCH):
def __init__(self, verbose=None, versions={}, ctype="deb", dist=ALL_DIST, arch=ALL_ARCH):
self.versions = versions
self.ctype = ctype
if not dist:
@ -33,6 +34,7 @@ class Component:
arch = ALL_ARCH
self.dist = dist
self.arch = arch
self.verbose = verbose
@classmethod
def get_versions(cls, version_file):
@ -51,7 +53,7 @@ class Component:
return result
def clone(self):
return Component(self.versions.copy(), self.ctype, self.dist, self.arch)
return Component(self.verbose, self.versions.copy(), self.ctype, self.dist, self.arch)
def merge(self, versions, overwritten=True):
for package in versions:
@ -71,7 +73,7 @@ class Component:
result.append(lines)
else:
result.append('{0}=={1}'.format(package, self.versions[package]))
return "\n".join(result)
return "\n".join(result)+'\n'
def dump_to_file(self, version_file, config=False, priority=999):
if len(self.versions) <= 0:
@ -92,6 +94,35 @@ class Component:
file_path = os.path.join(file_path, filename)
self.dump_to_file(file_path, config, priority)
def print(self, file_path):
if len(self.versions) <= 0:
return
if self.verbose is None:
return
filename = self.get_filename()
file_path = os.path.join(file_path, filename)
if self.verbose and re.search("cfile=", self.verbose) \
and not re.search(self.verbose, "cfile=all".format(filename)) \
and not re.search(self.verbose, "cfile={}".format(filename)):
return
print("VERSION : {}".format(file_path))
for package in sorted(self.versions.keys(), key=str.casefold):
if self.verbose and re.search("ctype=", self.verbose) \
and not re.search("ctype=all".format(self.ctype), self.verbose) \
and not re.search("ctype={}".format(self.ctype), self.verbose):
continue
if self.verbose and re.search("cname=", self.verbose) \
and not re.search(self.verbose, "cname=all".format(package)) \
and not re.search(self.verbose, "cname={}".format(package)):
continue
if self.verbose and re.search("cver=", self.verbose) \
and not re.search(self.verbose, "cver=all".format(self.versions[package])) \
and not re.search(self.verbose, "cver={}".format(self.versions[package])):
continue
print('{0}=={1}'.format(package, self.versions[package]))
# Check if the self component can be overwritten by the input component
def check_overwritable(self, component, for_all_dist=False, for_all_arch=False):
if self.ctype != component.ctype:
@ -153,9 +184,11 @@ class VersionModule:
name -- The name of the image, such as sonic-slave-buster, docker-lldp, etc
'''
def __init__(self, name=None, components=None):
def __init__(self, verbose=None, name=None, components=None):
self.name = name
self.components = components
self.module_path=""
self.verbose=verbose
# Overwrite the docker/host image/base image versions
def overwrite(self, module, for_all_dist=False, for_all_arch=False):
@ -191,6 +224,7 @@ class VersionModule:
module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS)
return self._get_config_module(module, dist, arch)
#Merge the default with specific version
def _get_config_module(self, default_module, dist, arch):
module = default_module.clone()
default_ctype_components = module._get_components_per_ctypes()
@ -205,11 +239,11 @@ class VersionModule:
continue
config_component = self._get_config_for_ctype(components, dist, arch)
config_components.append(config_component)
config_module = VersionModule(self.name, config_components)
config_module = VersionModule(self.verbose, self.name, config_components)
return config_module
def _get_config_for_ctype(self, components, dist, arch):
result = Component({}, components[0].ctype, dist, arch)
result = Component(self.verbose, {}, components[0].ctype, dist, arch)
for component in sorted(components, key = lambda x : x.get_order_keys()):
if result.check_inheritable(component):
result.merge(component.versions, True)
@ -224,7 +258,7 @@ class VersionModule:
components = sorted(components, key = lambda x : x.get_order_keys())
for i in range(0, len(components)):
component = components[i]
base_module = VersionModule(self.name, components[0:i])
base_module = VersionModule(self.verbose, self.name, components[0:i])
config_module = base_module._get_config_module(default_module, component.dist, component.arch)
config_components = config_module._get_components_by_ctype(ctype)
if len(config_components) > 0:
@ -253,7 +287,7 @@ class VersionModule:
result = []
for i in range(0, len(components)):
component = components[i]
inheritable_component = Component({}, component.ctype)
inheritable_component = Component(self.verbose, {}, component.ctype)
for j in range(0, i):
base_component = components[j]
if component.check_inheritable(base_component):
@ -276,6 +310,7 @@ class VersionModule:
file_paths = glob.glob(version_file_pattern)
components = []
self.name = os.path.basename(image_path)
self.module_path = image_path
self.components = components
for file_path in file_paths:
filename = os.path.basename(file_path)
@ -296,18 +331,25 @@ class VersionModule:
if filter_arch and arch and filter_arch != arch and arch != ALL_ARCH:
continue
versions = Component.get_versions(file_path)
component = Component(versions, ctype, dist, arch)
component = Component(self.verbose, versions, ctype, dist, arch)
components.append(component)
if self.verbose and re.search("stage=load", self.verbose):
component.print(file_path)
def load_from_target(self, image_path):
self.module_path=image_path
post_versions = os.path.join(image_path, 'post-versions')
if os.path.exists(post_versions):
self.load(post_versions)
self.name = os.path.basename(image_path)
if self.verbose and re.search("stage=post", self.verbose):
self.print(post_versions)
pre_versions = os.path.join(image_path, 'pre-versions')
if os.path.exists(pre_versions):
pre_module = VersionModule()
pre_module = VersionModule(self.verbose)
pre_module.load(pre_versions)
if self.verbose and re.search("stage=pre", self.verbose):
pre_module.print(pre_versions)
self.subtract(pre_module)
else:
self.load(image_path)
@ -319,6 +361,15 @@ class VersionModule:
for component in self.components:
component.dump_to_path(module_path, config, priority)
def print(self, module_path):
if self.verbose is None:
return
if re.search("cmod=", self.verbose) \
and not re.search(self.verbose, "cmod=all".format(self.name)) \
and not re.search(self.verbose, "cmod={}".format(self.name)):
return
for component in self.components:
component.print(module_path)
def filter(self, ctypes=[]):
if 'all' in ctypes:
return self
@ -340,7 +391,7 @@ class VersionModule:
if ctypes and component.ctype not in ctypes:
continue
components.append(component.clone())
return VersionModule(self.name, components)
return VersionModule(self.verbose, self.name, components)
def is_slave_module(self):
return self.name.startswith('sonic-slave-')
@ -370,14 +421,18 @@ class VersionModule:
return os.path.join(source_path, 'files/build/versions/build', module_name)
return os.path.join(source_path, 'files/build/versions/dockers', module_name)
def __repr__(self):
return repr(self.name)
class VersionBuild:
'''
The VersionBuild consists of multiple version modules.
'''
def __init__(self, target_path="./target", source_path='.'):
def __init__(self, verbose=None, target_path="./target", source_path='.'):
self.target_path = target_path
self.source_path = source_path
self.verbose = verbose
self.modules = {}
def load_from_target(self):
@ -394,8 +449,11 @@ class VersionBuild:
for file_path in file_paths:
if not os.path.isdir(file_path):
continue
module = VersionModule()
module = VersionModule(self.verbose)
module.load_from_target(file_path)
if self.verbose and re.search("stage=tmodname", self.verbose):
print("Target modname={}, path={}".format(module.name, file_path))
module.print(file_path)
modules[module.name] = module
self._merge_dgb_modules()
@ -411,8 +469,11 @@ class VersionBuild:
modules = {}
self.modules = modules
for image_path in paths:
module = VersionModule()
module = VersionModule(self.verbose)
module.load(image_path)
if self.verbose and re.search("stage=smodname", self.verbose):
print("Source modname={}, path={}".format(module.name, image_path))
module.print(image_path)
modules[module.name] = module
def overwrite(self, build, for_all_dist=False, for_all_arch=False):
@ -430,6 +491,13 @@ class VersionBuild:
module_path = self.get_module_path(module)
module.dump(module_path)
def print(self, message=None):
if self.verbose is None:
return
if message is not None:
print("[============={}===========]".format(message))
for module in [ self.modules[x] for x in (sorted(self.modules, key = lambda x : x)) ]:
module.print(module.module_path)
def subtract(self, default_module):
none_aggregatable_module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS)
for module in self.modules.values():
@ -455,19 +523,38 @@ class VersionBuild:
self.dump()
return
self.load_from_source()
if self.verbose and re.search("stage=init", self.verbose):
self.print("Initial Source")
default_module = self.modules.get(DEFAULT_MODULE, None)
target_build = VersionBuild(self.target_path, self.source_path)
if self.verbose and re.search("stage=init", self.verbose):
default_module.print("Default Module")
target_build = VersionBuild(self.verbose, self.target_path, self.source_path)
target_build.load_from_target()
target_build.filter(ctypes=ctypes)
if self.verbose and re.search("stage=init", self.verbose):
target_build.print("Initial Target")
if not default_module:
raise Exception("The default versions does not exist")
for module in target_build.modules.values():
for module in [ target_build.modules[x] for x in (sorted(target_build.modules, key = lambda x : x)) ] :
if module.is_individule_version():
continue
tmp_module = module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS)
default_module.overwrite(tmp_module, for_all_dist=True, for_all_arch=True)
if self.verbose and re.search("stage=tmp", self.verbose):
default_module.print("TMP DEFAULT MODULE")
target_build.subtract(default_module)
if self.verbose and re.search("stage=tmp", self.verbose):
target_build.print("After Subtract Target")
self.print("After Subtract Source")
self.overwrite(target_build, for_all_dist=for_all_dist, for_all_arch=for_all_arch)
if self.verbose and re.search("stage=add", self.verbose):
self.print("After Merge")
if not self.verbose or not re.search("dryrun", self.verbose):
self.dump()
def filter(self, ctypes=[]):
@ -485,14 +572,14 @@ class VersionBuild:
for dist in dists:
versions = self._get_versions(ctype, dist)
common_versions = self._get_common_versions(versions)
component = Component(common_versions, ctype, dist)
component = Component(self.verbose, common_versions, ctype, dist)
components.append(component)
else:
versions = self._get_versions(ctype)
common_versions = self._get_common_versions(versions)
component = Component(common_versions, ctype)
component = Component(self.verbose, common_versions, ctype)
components.append(component)
return VersionModule(DEFAULT_MODULE, components)
return VersionModule(self.verbose, DEFAULT_MODULE, components)
def get_aggregatable_modules(self):
modules = {}
@ -619,11 +706,13 @@ class VersionManagerCommands:
parser.add_argument('-d', '--for_all_dist', action='store_true', help='apply the versions for all distributions')
parser.add_argument('-a', '--for_all_arch', action='store_true', help='apply the versions for all architectures')
parser.add_argument('-c', '--ctypes', default='all', help='component types to freeze')
parser.add_argument('-v', '--verbose', default=None, help="verbose mode")
args = parser.parse_args(sys.argv[2:])
ctypes = args.ctypes.split(',')
if len(ctypes) == 0:
ctypes = ['all']
build = VersionBuild(target_path=args.target_path, source_path=args.source_path)
build = VersionBuild(verbose=args.verbose, target_path=args.target_path, source_path=args.source_path)
build.freeze(rebuild=args.rebuild, for_all_dist=args.for_all_dist, for_all_arch=args.for_all_arch, ctypes=ctypes)
def merge(self):
@ -632,6 +721,8 @@ class VersionManagerCommands:
parser.add_argument('-m', '--module_path', default=None, help='merge path, use the target path if not specified')
parser.add_argument('-b', '--base_path', required=True, help='base path, merge to the module path')
parser.add_argument('-e', '--exclude_module_path', default=None, help='exclude module path')
parser.add_argument('-i', '--include_module_path', default=None, help='include module path')
parser.add_argument('-v', '--verbose', default=None, help="verbose mode")
args = parser.parse_args(sys.argv[2:])
module_path = args.module_path
if not module_path:
@ -640,15 +731,22 @@ class VersionManagerCommands:
print('The module path {0} does not exist'.format(module_path))
if not os.path.exists(args.target_path):
os.makedirs(args.target_path)
module = VersionModule()
module = VersionModule(args.verbose)
module.load(module_path)
base_module = VersionModule()
base_module = VersionModule(args.verbose)
base_module.load(args.base_path)
module.overwrite(base_module)
if args.exclude_module_path:
exclude_module = VersionModule()
exclude_module = VersionModule(args.verbose)
exclude_module.load(args.exclude_module_path)
module.subtract(exclude_module)
if args.include_module_path:
include_module = VersionModule(args.verbose)
include_module.load(args.include_module_path)
if args.verbose:
include_module.print(args.include_module_path)
include_module.overwrite(module)
module.overwrite(include_module)
module.dump(args.target_path)
def generate(self):
@ -661,6 +759,7 @@ class VersionManagerCommands:
parser.add_argument('-d', '--distribution', required=True, help="distribution")
parser.add_argument('-a', '--architecture', required=True, help="architecture")
parser.add_argument('-p', '--priority', default=999, help="priority of the debian apt preference")
parser.add_argument('-v', '--verbose', default=None, help="verbose mode")
args = parser.parse_args(sys.argv[2:])
module_path = args.module_path
@ -668,11 +767,20 @@ class VersionManagerCommands:
module_path = VersionModule.get_module_path_by_name(args.source_path, args.module_name)
if not os.path.exists(args.target_path):
os.makedirs(args.target_path)
module = VersionModule()
module = VersionModule(args.verbose)
module.load(module_path, filter_dist=args.distribution, filter_arch=args.architecture)
config = module.get_config_module(args.source_path, args.distribution, args.architecture)
if args.verbose:
config.print(args.source_path)
config.clean_info(force=True)
config.dump(args.target_path, config=True, priority=args.priority)
if __name__ == "__main__":
VersionManagerCommands()
"""
Dry run examples:
scripts/versions_manager.py freeze -v 'dryrun|cmod=docker-config-engine-stretch|cfile=versions-py2|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp'
scripts/versions_manager.py freeze -v 'dryrun|cmod=default|cfile=versions-docker|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp'
"""

53
slave.mk
View File

@ -92,6 +92,10 @@ export BUILD_WORKDIR
## Define configuration, help etc.
###############################################################################
# Install the updated build hooks if INSHOOKS flag is set
export INSHOOKS=y
$(if $(INSHOOKS),$(shell sudo dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb &>/dev/null))
.platform :
ifneq ($(CONFIGURED_PLATFORM),generic)
$(Q)echo Build system is not configured, please run make configure
@ -423,8 +427,9 @@ $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))
endif
# Overwrite the buildinfo in slave container
$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV))
ifeq ($(filter clean,$(MAKECMDGOALS)),)
$(shell DBGOPT='$(DBGOPT)' scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV))
endif
include Makefile.cache
ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y)
@ -558,7 +563,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_ONLINE_DEBS)) : $(DEBS_PATH)/% : .platform \
if [ -z '$($*_CACHE_LOADED)' ] ; then
$(foreach deb,$* $($*_DERIVED_DEBS), \
{ curl -L -f -o $(DEBS_PATH)/$(deb) $($(deb)_CURL_OPTIONS) $($(deb)_URL) $(LOG) || { exit 1 ; } } ; )
{ SKIP_BUILD_HOOK=$($*_SKIP_VERSION) curl -L -f -o $(DEBS_PATH)/$(deb) $($(deb)_CURL_OPTIONS) $($(deb)_URL) $(LOG) || { exit 1 ; } } ; )
# Save the target deb into DPKG cache
$(call SAVE_CACHE,$*,$@)
@ -575,7 +580,7 @@ SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_ONLINE_DEBS))
# SONIC_ONLINE_FILES += $(SOME_NEW_FILE)
$(addprefix $(FILES_PATH)/, $(SONIC_ONLINE_FILES)) : $(FILES_PATH)/% : .platform
$(HEADER)
curl -L -f -o $@ $($*_CURL_OPTIONS) $($*_URL) $(LOG)
SKIP_BUILD_HOOK=$($*_SKIP_VERSION) curl -L -f -o $@ $($*_CURL_OPTIONS) $($*_URL) $(LOG)
$(FOOTER)
SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_ONLINE_FILES))
@ -883,7 +888,7 @@ $(SONIC_INSTALL_WHEELS) : $(PYTHON_WHEELS_PATH)/%-install : .platform $$(addsuff
while true; do
if mkdir $(PYTHON_WHEELS_PATH)/pip_lock &> /dev/null; then
ifneq ($(CROSS_BUILD_ENVIRON),y)
{ sudo -E pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; }
{ sudo -E SKIP_BUILD_HOOK=Y pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; }
else
# Link python script and data expected location to the cross python virtual env istallation locations
{ PATH=$(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION)):${PATH} sudo -E $(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION))/pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && $(if $(findstring $(SONIC_CONFIG_ENGINE_PY3),$*),(sudo ln -s $(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION))/sonic-cfggen /usr/local/bin/sonic-cfggen 2>/dev/null || true), true ) && $(if $(findstring $(SONIC_YANG_MODELS_PY3),$*),(sudo ln -s $(VIRTENV_BASE_CROSS_PYTHON3)/yang-models /usr/local/yang-models 2>/dev/null || true), true ) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; }
@ -911,7 +916,11 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
# Apply series of patches if exist
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && QUILT_PATCHES=../$(notdir $($*.gz_PATH)).patch quilt push -a; popd; fi
# Prepare docker build info
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo
SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo $(LOG)
docker info $(LOG)
docker build --squash --no-cache \
--build-arg http_proxy=$(HTTP_PROXY) \
@ -926,9 +935,12 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH} \
DBGOPT='$(DBGOPT)' \
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
$(FOOTER)
@ -1030,7 +1042,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH)
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(LOG)
docker info $(LOG)
docker build --squash --no-cache \
--build-arg http_proxy=$(HTTP_PROXY) \
@ -1042,6 +1056,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
--build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \
--build-arg frr_user_uid=$(FRR_USER_UID) \
--build-arg frr_user_gid=$(FRR_USER_GID) \
--build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
--build-arg SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
--build-arg image_version=$(SONIC_IMAGE_VERSION) \
--label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \
--label Tag=$(SONIC_IMAGE_VERSION) \
@ -1049,10 +1065,13 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
DBGOPT='$(DBGOPT)' \
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@ -1092,7 +1111,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH)
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
scripts/prepare_docker_buildinfo.sh $*-dbg $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) $(LOG)
docker info $(LOG)
docker build \
$(if $($*.gz_DBG_DEPENDS), --squash --no-cache, --no-cache) \
@ -1100,16 +1121,21 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
--build-arg https_proxy=$(HTTPS_PROXY) \
--build-arg no_proxy=$(NO_PROXY) \
--build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \
--build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
--build-arg SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
--label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \
--label Tag=$(SONIC_IMAGE_VERSION) \
--file $($*.gz_PATH)/Dockerfile-dbg \
-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
DBGOPT='$(DBGOPT)' \
scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
# Clean up
docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@ -1398,6 +1424,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
SIGNING_KEY="$(SIGNING_KEY)" \
SIGNING_CERT="$(SIGNING_CERT)" \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
DBGOPT='$(DBGOPT)' \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \
@ -1466,7 +1494,7 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
$(SONIC_SIMPLE_DOCKER_IMAGES) \
$(SONIC_INSTALLERS)))
$(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
$(Q)rm -f $(TARGET_PATH)/$*
$(Q)rm -f $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*)
SONIC_CLEAN_STDEB_DEBS = $(addsuffix -clean,$(addprefix $(PYTHON_DEBS_PATH)/, \
$(SONIC_PYTHON_STDEB_DEBS)))
@ -1480,8 +1508,13 @@ $(SONIC_CLEAN_WHEELS) :: $(PYTHON_WHEELS_PATH)/%-clean : .platform
clean-logs :: .platform
$(Q)rm -f $(TARGET_PATH)/*.log $(DEBS_PATH)/*.log $(FILES_PATH)/*.log $(PYTHON_DEBS_PATH)/*.log $(PYTHON_WHEELS_PATH)/*.log
clean-versions :: .platform
@rm -rf target/versions/*
clean :: .platform clean-logs $$(SONIC_CLEAN_DEBS) $$(SONIC_CLEAN_FILES) $$(SONIC_CLEAN_TARGETS) $$(SONIC_CLEAN_STDEB_DEBS) $$(SONIC_CLEAN_WHEELS)
vclean:: .platform
@sudo rm -rf target/vcache/* target/baseimage*
clean :: .platform clean-logs clean-versions $$(SONIC_CLEAN_DEBS) $$(SONIC_CLEAN_FILES) $$(SONIC_CLEAN_TARGETS) $$(SONIC_CLEAN_STDEB_DEBS) $$(SONIC_CLEAN_WHEELS)
###############################################################################
## all

2
src/sonic-build-hooks/Makefile
View File

@ -5,7 +5,7 @@ SONIC_BUILD_HOOKS_PACKAGE = $(SONIC_BUILD_HOOKS)_$(SONIC_BUILD_HOOKS_VERSION)_al
BUILDINFO_DIR = buildinfo
TMP_DIR = tmp
SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts
SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files
SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files post_run_cleanup
SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE)
BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS)
DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN

45
src/sonic-build-hooks/scripts/buildinfo_base.sh
View File

@ -23,6 +23,7 @@ fi
PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz
mkdir -p ${PKG_CACHE_PATH}
. ${BUILDINFO_PATH}/scripts/utils.sh
URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#")
@ -35,9 +36,15 @@ fi
log_err()
{
echo "$1" >> $LOG_PATH/error.log
echo "$(date "+%F-%H-%M-%S") ERR $1" >> $LOG_PATH/error.log
echo "$1" 1>&2
}
log_info()
{
echo "$(date "+%F-%H-%M-%S") INFO $1" >> $LOG_PATH/info.log
echo "$1" 1>&2
}
# Get the real command not hooked by sonic-build-hook package
get_command()
@ -76,6 +83,28 @@ check_if_url_exist()
fi
}
get_version_cache_option()
{
#SONIC_VERSION_CACHE="cache"
if [ ! -z ${SONIC_VERSION_CACHE} ]; then
if [ ${SONIC_VERSION_CACHE} == "rcache" ]; then
echo -n "rcache"
elif [ ${SONIC_VERSION_CACHE} == "wcache" ]; then
echo -n "wcache"
elif [ ${SONIC_VERSION_CACHE} == "cache" ]; then
echo -n "wcache"
else
echo -n ""
return 1
fi
echo -n ""
return 0
fi
echo -n ""
return 1
}
# Enable or disable the reproducible mirrors
set_reproducible_mirrors()
{
@ -115,7 +144,7 @@ download_packages()
local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
[ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
if [ -z "$version" ]; then
echo "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
continue
fi
@ -129,7 +158,7 @@ download_packages()
else
real_version=$(get_url_version $url)
if [ "$real_version" != "$version" ]; then
echo "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
exit 1
fi
fi
@ -294,10 +323,10 @@ update_version_file()
if [ ! -f "$pre_version_file" ]; then
return 0
fi
local pacakge_versions="$(cat $pre_version_file)"
[ -f "$version_file" ] && pacakge_versions="$pacakge_versions $(cat $version_file)"
local package_versions="$(cat $pre_version_file)"
[ -f "$version_file" ] && package_versions="$package_versions $(cat $version_file)"
declare -A versions
for pacakge_version in $pacakge_versions; do
for pacakge_version in $package_versions; do
package=$(echo $pacakge_version | awk -F"==" '{print $1}')
version=$(echo $pacakge_version | awk -F"==" '{print $2}')
if [ -z "$package" ] || [ -z "$version" ]; then
@ -331,4 +360,8 @@ ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2")
ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3")
ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web")
ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git")
ENABLE_VERSION_CONTROL_PIP=$(check_version_control "pip")
ENABLE_VERSION_CONTROL_PYTHON=$(check_version_control "python")
ENABLE_VERSION_CONTROL_EASY_INSTALL=$(check_version_control "easy_install")
ENABLE_VERSION_CONTROL_GO=$(check_version_control "go")
ENABLE_VERSION_CONTROL_DOCKER=$(check_version_control "docker")

15
src/sonic-build-hooks/scripts/collect_version_files
View File

@ -1,8 +1,9 @@
#!/bin/bash
TARGET_PATH=$1
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
TARGET_PATH=$1
[ -d ${TARGET_PATH} ] && rm -rf ${TARGET_PATH}
[ -z "$TARGET_PATH" ] && TARGET_PATH=$POST_VERSION_PATH
ARCH=$(dpkg --print-architecture)
DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)
@ -11,9 +12,15 @@ DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)
mkdir -p $TARGET_PATH
chmod a+rw $TARGET_PATH
dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze >> "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze >> "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
# Skip the package that does have a static build version.
# SAI package versions are changed too frequently.
SKIP_VERSION_PACKAGE="libsaibcm|libpaibcm|linuxptp|@ file://"
dpkg-query -W -f '${Package}==${Version}\n' | grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
[ -f "${BUILD_WEB_VERSION_FILE}" ] && cp ${BUILD_WEB_VERSION_FILE} ${TARGET_PATH}
[ -f "${BUILD_GIT_VERSION_FILE}" ] && cp ${BUILD_GIT_VERSION_FILE} ${TARGET_PATH}
## Add the the packages purged
[ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"

18
src/sonic-build-hooks/scripts/post_run_buildinfo
View File

@ -4,15 +4,23 @@ IMAGENAME=$1
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
[ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH
# Collect the version files
collect_version_files $POST_VERSION_PATH
#Save the cache file for exporting it to host.
tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
# Disable the build hooks
symlink_build_hooks -d
set_reproducible_mirrors -d
[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
rm -rf $BUILD_VERSION_PATH/*
if [ ! -z "$(get_version_cache_option)" ]; then
# Restore he deletion of cache files
cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
EOF
fi
# Remove the version deb preference
rm -f $VERSION_DEB_PREFERENCE

40
src/sonic-build-hooks/scripts/post_run_cleanup Executable file
View File

@ -0,0 +1,40 @@
#!/bin/bash
IMAGENAME=$1
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
set -x
if [ ! -z "$(get_version_cache_option)" ]; then
#Delete the rsync package files
if [[ ! ${IMAGENAME} =~ -slave- ]]; then
/usr/bin/apt-get purge -y --auto-remove rsync
fi
fi
apt-get -s clean -y
apt-get -s autoclean -y
apt-get -s autoremove -y
#apt-get -s autoremove -y --purge
rm -f /var/cache/apt/archives/*.deb /var/cache/apt/*.bin
if [[ ! ${IMAGENAME} =~ -slave- ]]; then
rm -f /var/lib/apt/lists/*
fi
rm -rf /sonic/target /ssh
rm -f /tmp/*
rm -rf /debs /python-wheels ~/.cache
find / | grep -E "__pycache__" | xargs rm -rf
rm -rf $BUILD_VERSION_PATH/*
# Disable the build hooks
symlink_build_hooks -d
set_reproducible_mirrors -d
# Remove the version deb preference
rm -f $VERSION_DEB_PREFERENCE

16
src/sonic-build-hooks/scripts/pre_run_buildinfo
View File

@ -18,6 +18,22 @@ symlink_build_hooks
set_reproducible_mirrors
mkdir -p /var/cache/apt/archives/
mkdir -p ${PKG_CACHE_PATH}/deb/
if [ ! -z "$(get_version_cache_option)" ]; then
# Skip the deletion of cache files
cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
DPkg::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
APT::Update::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
APT::Keep-Downloaded-Packages "true";
EOF
fi
# Extract the cache inside the docker build.
if [ -f ${PKG_CACHE_FILE_NAME} ]; then
tar -C ${PKG_CACHE_PATH} -xvf ${PKG_CACHE_FILE_NAME}
test -e ${PKG_CACHE_PATH}/deb && cp ${PKG_CACHE_PATH}/deb/* /var/cache/apt/archives/
fi
chmod -R a+rw $BUILDINFO_PATH

37
src/sonic-build-hooks/scripts/utils.sh Normal file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# Lock macro for shared file access
# Parameters:
# $(1) - Lock file name
# $(2) - Timeout value
function FLOCK()
{
local filename=$(dirname $1)
local timeout=${2:-360}
if [[ ! -f ${filename}.flock ]]; then
touch ${filename}.flock
chmod -f 777 ${filename}.flock;
fi
local lockname=$(basename ${filename})
local lock_fd=lock_${lockname//[%.\/\-+~]/_}_fd
eval $(echo exec {${lock_fd}}\<\>"${filename}.flock")
#echo ${!lock_fd}
if ! flock -x -w ${timeout} "${!lock_fd}" ; then
echo "ERROR: Lock timeout trying to access ${filename}.flock" 1>&2;
exit 1;
fi
#echo "Lock acquired .."
}
# UnLock macro for shared file access
# Parameters:
# $(1) - Lock file name
function FUNLOCK()
{
local filename=$(dirname $1)
local lockname=$(basename ${filename})
local lock_fd=lock_${lockname//[%.\/\-+~]/_}_fd
eval $(echo exec "${!lock_fd}<&-")
#rm -f ${filename}.flock
}