matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[build]: wait for conflicts package to be uninstalled (#5039)

Browse Source 
when parallel build is enabled, both docker-fpm-frr and docker-syncd-brcm
is built at the same time, docker-fpm-frr requires swss which requires to
install libsaivs-dev. docker-syncd-brcm requires syncd package which requires
to install libsaibcm-dev.

since libsaivs-dev and libsaibcm-dev install the sai header in the same
location, these two packages cannot be installed at the same time. Therefore,
we need to serialize the build between these two packages. Simply uninstall
the conflict package is not enough to solve this issue. The correct solution
is to have one package wait for another package to be uninstalled.

For example, if syncd is built first, then it will install libsaibcm-dev.
Meanwhile, if the swss build job starts and tries to install libsaivs-dev,
it will first try to query if libsaibcm-dev is installed or not. if it is
installed, then it will wait until libsaibcm-dev is uninstalled. After syncd
job is finished, it will uninstall libsaibcm-dev and swss build job will be
unblocked.

To solve this issue, _UNINSTALLS is introduced to uninstall a package that
is no longer needed and to allow blocked job to continue.

Signed-off-by: Guohan Lu <lguohan@gmail.com>
This commit is contained in:
lguohan 2020-07-27 10:46:20 -07:00 committed by GitHub
parent c0d1616f89
commit e1ac3cfc6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 76 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
platform/barefoot/bfn-sai.mk
View File

@ -2,8 +2,8 @@ BFN_SAI = bfnsdk_20200721_deb9.deb
$(BFN_SAI)_URL = "https://github.com/barefootnetworks/sonic-release-pkgs/raw/dev/$(BFN_SAI)"
$(BFN_SAI)_DEPENDS += $(LIBNL_GENL3_DEV)
$(eval $(call add_conflict_package,$(BFN_SAI),$(LIBSAIVS_DEV)))
$(BFN_SAI)_RDEPENDS += $(LIBNL_GENL3)
SONIC_ONLINE_DEBS += $(BFN_SAI)
$(BFN_SAI_DEV)_DEPENDS += $(BFN_SAI)
$(BFN_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV)

1
platform/barefoot/rules.mk
View File

@ -22,6 +22,7 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(SONIC_ONE_ABOOT) \
# Inject sai into syncd
#$(SYNCD)_DEPENDS += $(BFN_SAI) $(WNC_OSW1800_PLATFORM) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM)
$(SYNCD)_DEPENDS += $(BFN_SAI) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM)
$(SYNCD)_UNINSTALLS += $(BFN_SAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)

2
platform/broadcom/rules.mk
View File

@ -34,6 +34,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(SONIC_ONE_ABOOT_IMAGE) \
# Inject brcm sai into syncd
$(SYNCD)_DEPENDS += $(BRCM_SAI) $(BRCM_SAI_DEV)
$(SYNCD)_UNINSTALLS += $(BRCM_SAI_DEV)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

2
platform/broadcom/sai.mk
View File

@ -6,4 +6,4 @@ $(BRCM_SAI_DEV)_URL = "https://sonicstorage.blob.core.windows.net/packages/bcmsa
SONIC_ONLINE_DEBS += $(BRCM_SAI)
$(BRCM_SAI_DEV)_DEPENDS += $(BRCM_SAI)
$(BRCM_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(BRCM_SAI_DEV),$(LIBSAIVS_DEV)))

2
platform/cavium/rules.mk
View File

@ -14,6 +14,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \
# Inject cavium sai into syncd
$(SYNCD)_DEPENDS += $(CAVM_SAI) $(CAVM_LIBSAI)
$(SYNCD)_UNINSTALLS += $(CAVM_LIBSAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV_CAVM)
endif

2
platform/centec/rules.mk
View File

@ -11,6 +11,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE)
# Inject centec sai into syncd
$(SYNCD)_DEPENDS += $(CENTEC_SAI)
$(SYNCD)_UNINSTALLS += $(CENTEC_SAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

2
platform/centec/sdk.mk
View File

@ -1,6 +1,6 @@
# Centec SAI
CENTEC_SAI = libsai_1.6.3-1_amd64.deb
$(CENTEC_SAI)_URL = https://github.com/CentecNetworks/sonic-binaries/raw/master/amd64/$(CENTEC_SAI)
$(CENTEC_SAI)_CONFLICTS += $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(CENTEC_SAI),$(LIBSAIVS_DEV)))
SONIC_ONLINE_DEBS += $(CENTEC_SAI)

2
platform/innovium/invm-sai.mk
View File

@ -10,6 +10,6 @@ $(INVM_LIBSAI)_URL = $(INVM_SAI_ONLINE)/$(INVM_LIBSAI)
$(INVM_HSAI)_URL = $(INVM_SAI_ONLINE)/$(INVM_HSAI)
$(INVM_DRV)_URL = $(INVM_SAI_ONLINE)/$(INVM_DRV)
$(INVM_HSAI)_CONFLICTS += $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(INVM_HSAI),$(LIBSAIVS_DEV)))
SONIC_ONLINE_DEBS += $(INVM_LIBSAI) $(INVM_HSAI) $(INVM_DRV)

1
platform/innovium/rules.mk
View File

@ -15,6 +15,7 @@ SONIC_ALL += $(SONIC_INVM_ONE_IMAGE) \
# Inject invm sai into syncd
$(SYNCD)_DEPENDS += $(INVM_HSAI) $(INVM_LIBSAI) $(LIBSAITHRIFT_DEV_INVM)
$(SYNCD)_UNINSTALLS += $(INVM_HSAI)
# Runtime dependency on invm sai is set only for syncd
$(SYNCD)_RDEPENDS += $(INVM_HSAI)

2
platform/marvell-arm64/rules.mk
View File

@ -17,6 +17,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \
# Inject mrvl sai into syncd
$(SYNCD)_DEPENDS += $(MRVL_SAI)
$(SYNCD)_UNINSTALLS += $(MRVL_DEV)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

3
platform/marvell-arm64/sai.mk
View File

@ -4,5 +4,6 @@ export MRVL_SAI_VERSION = 1.5.1
export MRVL_SAI = mrvllibsai_$(PLATFORM_ARCH)_$(MRVL_SAI_VERSION).deb
$(MRVL_SAI)_SRC_PATH = $(PLATFORM_PATH)/sai
$(MRVL_SAI)_CONFLICTS = $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(MRVL_SAI),$(LIBSAIVS_DEV)))
SONIC_MAKE_DEBS += $(MRVL_SAI)

2
platform/marvell-armhf/rules.mk
View File

@ -18,6 +18,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \
# Inject mrvl sai into syncd
$(SYNCD)_DEPENDS += $(MRVL_SAI)
$(SYNCD)_UNINSTALLS += $(MRVL_SAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

3
platform/marvell-armhf/sai.mk
View File

@ -4,5 +4,6 @@ export MRVL_SAI_VERSION = 1.5.1
export MRVL_SAI = mrvllibsai_$(PLATFORM_ARCH)_$(MRVL_SAI_VERSION).deb
$(MRVL_SAI)_SRC_PATH = $(PLATFORM_PATH)/sai
$(MRVL_SAI)_CONFLICTS = $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(MRVL_SAI),$(LIBSAIVS_DEV)))
SONIC_MAKE_DEBS += $(MRVL_SAI)

2
platform/marvell/rules.mk
View File

@ -10,6 +10,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \
# Inject mrvl sai into syncd
$(SYNCD)_DEPENDS += $(MRVL_FPA) $(MRVL_SAI)
$(SYNCD)_UNINSTALLS += $(MRVL_SAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

2
platform/marvell/sai.mk
View File

@ -4,6 +4,6 @@ export MRVL_SAI_VERSION = 1.5.1
export MRVL_SAI = mrvllibsai_amd64_$(MRVL_SAI_VERSION).deb
$(MRVL_SAI)_SRC_PATH = $(PLATFORM_PATH)/sai
$(MRVL_SAI)_CONFLICTS = $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(MRVL_SAI),$(LIBSAIVS_DEV)))
SONIC_MAKE_DEBS += $(MRVL_SAI)

2
platform/mellanox/mlnx-sai.mk
View File

@ -8,7 +8,7 @@ MLNX_SAI = mlnx-sai_1.mlnx.$(MLNX_SAI_VERSION)_amd64.deb
$(MLNX_SAI)_SRC_PATH = $(PLATFORM_PATH)/mlnx-sai
$(MLNX_SAI)_DEPENDS += $(MLNX_SDK_DEBS)
$(MLNX_SAI)_RDEPENDS += $(MLNX_SDK_RDEBS) $(MLNX_SDK_DEBS)
$(MLNX_SAI)_CONFLICTS += $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(MLNX_SAI),$(LIBSAIVS_DEV)))
MLNX_SAI_DBGSYM = mlnx-sai-dbgsym_1.mlnx.$(MLNX_SAI_VERSION)_amd64.deb
$(eval $(call add_derived_package,$(MLNX_SAI),$(MLNX_SAI_DBGSYM)))
SONIC_MAKE_DEBS += $(MLNX_SAI)

2
platform/mellanox/rules.mk
View File

@ -20,6 +20,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \
# Inject mlnx sai into syncd
$(SYNCD)_DEPENDS += $(MLNX_SAI)
$(SYNCD)_UNINSTALLS += $(MLNX_SAI)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

2
platform/nephos/rules.mk
View File

@ -24,6 +24,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(DOCKER_FPM)
# Inject nephos sai into syncd
$(SYNCD)_DEPENDS += $(NEPHOS_SAI) $(NEPHOS_SAI_DEV)
$(SYNCD)_UNINSTALLS += $(NEPHOS_SAI_DEV)
ifeq ($(ENABLE_SYNCD_RPC),y)
$(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV)
endif

2
platform/nephos/sai.mk
View File

@ -33,4 +33,4 @@ else
SONIC_ONLINE_DEBS += $(NEPHOS_SAI) $(NEPHOS_SAI_DEV)
endif
$(NEPHOS_SAI_DEV)_DEPENDS += $(NEPHOS_SAI)
$(NEPHOS_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV)
$(eval $(call add_conflict_package,$(NEPHOS_SAI_DEV),$(LIBSAIVS_DEV)))

27
rules/functions
View File

@ -96,6 +96,20 @@ $(1)_EXTRA_DEBS += $(2)
SONIC_EXTRA_DEBS += $(2)
endef
###############################################################################
## Definition of conflict packages
###############################################################################
# call:
# add_conflict_package some_deb.deb, conflict_deb
define add_conflict_package
$(1)_CONFLICT_DEBS += $(2)
$(2)_CONFLICT_DEBS += $(1)
endef
###############################################################################
## Definition of debug dockers
###############################################################################
# call:
# add_dbg_docker some_docker.gz, some-docker-dbg.gz
@ -126,3 +140,16 @@ endef
rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1)
###############################################################################
## Uninstall debs
###############################################################################
define UNINSTALL_DEBS
$(info $(1))
$(foreach deb,$(1), \
{ while true; do \
if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then \
{ sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(firstword $(subst _, ,$(basename $(deb)))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } \
fi; \
done; } )
endef

1
rules/sairedis.mk
View File

@ -16,6 +16,7 @@ $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS)))
LIBSAIVS_DEV = libsaivs-dev_1.0.0_$(CONFIGURED_ARCH).deb
$(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS_DEV)))
$(LIBSAIVS_DEV)_DEPENDS += $(LIBSAIVS)
LIBSAIMETADATA = libsaimetadata_1.0.0_$(CONFIGURED_ARCH).deb
$(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIMETADATA)))

1
rules/swss.mk
View File

@ -5,6 +5,7 @@ $(SWSS)_SRC_PATH = $(SRC_PATH)/sonic-swss
$(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \
$(LIBTEAMDCTL) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \
$(LIBSAIVS) $(LIBSAIVS_DEV)
$(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV)
$(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) \
$(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON_SWSSCOMMON)

44
slave.mk
View File

@ -355,6 +355,10 @@ $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES)) : $(FILES_PATH)/% : .platform $
$(call SAVE_CACHE,$*,$@)
fi
# Uninstall unneeded build dependency
$(call UNINSTALL_DEBS,$($*_UNINSTALLS))
$(FOOTER)
SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES))
@ -395,6 +399,9 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
fi
# Uninstall unneeded build dependency
$(call UNINSTALL_DEBS,$($*_UNINSTALLS))
$(FOOTER)
SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS))
@ -436,6 +443,9 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
$(call SAVE_CACHE,$*,$@)
fi
# Uninstall unneeded build dependency
$(call UNINSTALL_DEBS,$($*_UNINSTALLS))
$(FOOTER)
SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS))
@ -479,35 +489,17 @@ SONIC_INSTALL_TARGETS = $(addsuffix -install,$(addprefix $(DEBS_PATH)/, \
$(SONIC_PYTHON_STDEB_DEBS) \
$(SONIC_DERIVED_DEBS) \
$(SONIC_EXTRA_DEBS)))
$(SONIC_INSTALL_TARGETS) : $(DEBS_PATH)/%-install : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) $(DEBS_PATH)/$$* $$(addsuffix -uninstall,$$(addprefix $(DEBS_PATH)/,$$($$*_CONFLICTS)))
$(SONIC_INSTALL_TARGETS) : $(DEBS_PATH)/%-install : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) $(DEBS_PATH)/$$*
$(HEADER)
[ -f $(DEBS_PATH)/$* ] || { echo $(DEBS_PATH)/$* does not exist $(LOG) && false $(LOG) }
# put a lock here because dpkg does not allow installing packages in parallel
while true; do
if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then
{ sudo DEBIAN_FRONTEND=noninteractive dpkg -i $(DEBS_PATH)/$* $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; }
fi
done
$(FOOTER)
# Targets for installing debian packages prior to build one that depends on them
SONIC_UNINSTALL_TARGETS = $(addsuffix -uninstall,$(addprefix $(DEBS_PATH)/, \
$(SONIC_ONLINE_DEBS) \
$(SONIC_COPY_DEBS) \
$(SONIC_MAKE_DEBS) \
$(SONIC_DPKG_DEBS) \
$(SONIC_PYTHON_STDEB_DEBS) \
$(SONIC_DERIVED_DEBS) \
$(SONIC_EXTRA_DEBS)))
$(SONIC_UNINSTALL_TARGETS) : $(DEBS_PATH)/%-uninstall : .platform
$(HEADER)
[ -f $(DEBS_PATH)/$* ] || { echo $(DEBS_PATH)/$* does not exist $(LOG) && false $(LOG) }
# put a lock here because dpkg does not allow installing packages in parallel
while true; do
if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then
{ sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(firstword $(subst _, ,$(basename $*))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; }
fi
# wait for conflicted packages to be uninstalled
$(foreach deb, $($*_CONFLICT_DEBS), \
{ while dpkg -s $(firstword $(subst _, ,$(basename $(deb)))) &> /dev/null; do echo "waiting for $(deb) to be uninstalled" $(LOG); sleep 1; done } )
# put a lock here because dpkg does not allow installing packages in parallel
if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then
{ sudo DEBIAN_FRONTEND=noninteractive dpkg -i $(DEBS_PATH)/$* $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; }
fi
done
$(FOOTER)