[build] Add retry when make SONiC image to improve success rate. (#12325)
Why I did it Makefile needs some dependencies from the Internet. It will fail for network related issue. Retries will fix most of these issues. How I did it Add retries when running commands which maybe related with networking. How to verify it
This commit is contained in:
parent
025a8455cd
commit
9bec8bd198
@ -2,3 +2,5 @@ variables:
|
||||
DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
|
||||
COMMON_LIB_BUILD_ENVS: 'bullseye'
|
||||
SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
|
||||
SONIC_BUILD_RETRY_COUNT: 3
|
||||
SONIC_BUILD_RETRY_INTERVAL: 600
|
||||
|
9
Makefile
9
Makefile
@ -33,20 +33,21 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM))
|
||||
PLATFORM_CHECKOUT := platform/checkout
|
||||
PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini
|
||||
PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi)
|
||||
MAKE_WITH_RETRY := ./scripts/run_with_retry $(MAKE)
|
||||
|
||||
%::
|
||||
@echo "+++ --- Making $@ --- +++"
|
||||
ifeq ($(NOJESSIE), 0)
|
||||
EXTRA_DOCKER_TARGETS=$(notdir $@) $(MAKE) -f Makefile.work jessie
|
||||
$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) -f Makefile.work jessie
|
||||
endif
|
||||
ifeq ($(NOSTRETCH), 0)
|
||||
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch $(MAKE) -f Makefile.work stretch
|
||||
$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch -f Makefile.work stretch
|
||||
endif
|
||||
ifeq ($(NOBUSTER), 0)
|
||||
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster $(MAKE) -f Makefile.work buster
|
||||
$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster -f Makefile.work buster
|
||||
endif
|
||||
ifeq ($(NOBULLSEYE), 0)
|
||||
BLDENV=bullseye $(MAKE) -f Makefile.work $@
|
||||
$(MAKE_WITH_RETRY) BLDENV=bullseye -f Makefile.work $@
|
||||
endif
|
||||
BLDENV=bullseye $(MAKE) -f Makefile.work docker-cleanup
|
||||
|
||||
|
17
scripts/run_with_retry
Executable file
17
scripts/run_with_retry
Executable file
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
run_with_retry(){
|
||||
[ "$SONIC_BUILD_RETRY_COUNT" -gt 0 ] || SONIC_BUILD_RETRY_COUNT=0
|
||||
[[ "$*" == "" ]] && { echo "run_with_retry: input command can't be empty." 1>&2;exit 1; }
|
||||
for ((i=0; i<=$SONIC_BUILD_RETRY_COUNT; i++))
|
||||
do
|
||||
if [[ $i != 0 ]];then
|
||||
echo "==============================================================================" 1>&2
|
||||
echo "Waiting $SONIC_BUILD_RETRY_INTERVAL to run again, $i/$SONIC_BUILD_RETRY_COUNT" 1>&2
|
||||
echo "==============================================================================" 1>&2
|
||||
sleep $SONIC_BUILD_RETRY_INTERVAL
|
||||
fi
|
||||
"$@" && break
|
||||
done
|
||||
}
|
||||
run_with_retry "$@"
|
Loading…
Reference in New Issue
Block a user