matthew/sonic-buildimage
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[build] Add retry when make SONiC image to improve success rate. (#12325) (#13128)

Browse Source 
Why I did it
Makefile needs some dependencies from the Internet. It will fail for network related issue.
Retries will fix most of these issues.

How I did it
Add retries when running commands which maybe related with networking.

How to verify it
This commit is contained in:
Liu Shilong 2022-12-23 18:51:23 +08:00 committed by GitHub
parent 4b5d019bbe
commit 50e68daf50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.azure-pipelines/template-variables.yml
View File

@ -2,3 +2,5 @@ variables:
DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io' DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
COMMON_LIB_BUILD_ENVS: 'bullseye' COMMON_LIB_BUILD_ENVS: 'bullseye'
SONIC_SLAVE_DOCKER_DRIVER: 'overlay2' SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
SONIC_BUILD_RETRY_COUNT: 3
SONIC_BUILD_RETRY_INTERVAL: 600

9
Makefile
View File

@ -25,20 +25,21 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM))
PLATFORM_CHECKOUT := platform/checkout PLATFORM_CHECKOUT := platform/checkout
PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini
PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi) PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi)
MAKE_WITH_RETRY := ./scripts/run_with_retry $(MAKE)
%:: %::
@echo "+++ --- Making $@ --- +++" @echo "+++ --- Making $@ --- +++"
ifeq ($(NOJESSIE), 0) ifeq ($(NOJESSIE), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) make -f Makefile.work jessie $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) -f Makefile.work jessie
endif endif
ifeq ($(NOSTRETCH), 0) ifeq ($(NOSTRETCH), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch make -f Makefile.work stretch $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch -f Makefile.work stretch
endif endif
ifeq ($(NOBUSTER), 0) ifeq ($(NOBUSTER), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster make -f Makefile.work buster $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster -f Makefile.work buster
endif endif
ifeq ($(NOBULLSEYE), 0) ifeq ($(NOBULLSEYE), 0)
BLDENV=bullseye make -f Makefile.work $@ $(MAKE_WITH_RETRY) BLDENV=bullseye -f Makefile.work $@
endif endif
BLDENV=bullseye make -f Makefile.work docker-cleanup BLDENV=bullseye make -f Makefile.work docker-cleanup

17
scripts/run_with_retry Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
run_with_retry(){
[ "$SONIC_BUILD_RETRY_COUNT" -gt 0 ] || SONIC_BUILD_RETRY_COUNT=0
[[ "$*" == "" ]] && { echo "run_with_retry: input command can't be empty." 1>&2;exit 1; }
for ((i=0; i<=$SONIC_BUILD_RETRY_COUNT; i++))
do
if [[ $i != 0 ]];then
echo "==============================================================================" 1>&2
echo "Waiting $SONIC_BUILD_RETRY_INTERVAL to run again, $i/$SONIC_BUILD_RETRY_COUNT" 1>&2
echo "==============================================================================" 1>&2
sleep $SONIC_BUILD_RETRY_INTERVAL
fi
"$@" && break
done
}
run_with_retry "$@"