2016-12-05 13:12:19 -06:00
|
|
|
###############################################################################
|
|
|
|
|
## FUNCTIONS
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Colored output
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
# Enable colored output
|
|
|
|
|
ifeq ($(SONIC_CONFIG_ENABLE_COLORS),y)
|
|
|
|
|
ifeq ($(MAKE_TERMOUT),)
|
|
|
|
|
RED=\033[1;31m
|
|
|
|
|
PURPLE=\033[1;35m
|
|
|
|
|
CYAN=\033[1;36m
|
|
|
|
|
GREEN=\033[1;32m
|
|
|
|
|
GRAY=\033[0m
|
|
|
|
|
endif
|
|
|
|
|
endif
|
|
|
|
|
|
2020-03-23 11:45:43 -05:00
|
|
|
ifeq ($(BUILD_LOG_TIMESTAMP),simple)
|
2020-03-21 16:21:26 -05:00
|
|
|
PROCESS_LOG_OPTION = -t
|
|
|
|
|
endif
|
|
|
|
|
|
2016-12-05 13:12:19 -06:00
|
|
|
# Print red colored output
|
|
|
|
|
# call:
|
|
|
|
|
# log_red message
|
|
|
|
|
log_red = echo -e "$(RED)$(1)$(GRAY)"
|
|
|
|
|
|
|
|
|
|
# Print purple colored output
|
|
|
|
|
# call:
|
|
|
|
|
# log_purple message
|
|
|
|
|
log_purple = echo -e "$(PURPLE)$(1)$(GRAY)"
|
|
|
|
|
|
|
|
|
|
# Print blue colored output
|
|
|
|
|
# call:
|
|
|
|
|
# log_blue message
|
|
|
|
|
log_blue = echo -e "$(CYAN)$(1)$(GRAY)"
|
|
|
|
|
|
|
|
|
|
# Print green colored output
|
|
|
|
|
# call:
|
|
|
|
|
# log_green message
|
|
|
|
|
log_green = echo -e "$(GREEN)$(1)$(GRAY)"
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Logging
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
FLUSH_LOG = rm -f $@.log
|
|
|
|
|
|
2020-03-21 16:21:26 -05:00
|
|
|
LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
|
2021-01-28 04:21:38 -06:00
|
|
|
LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; test $${PIPESTATUS[-2]} -eq 0 || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
|
2016-12-05 13:12:19 -06:00
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Header and footer for each target
|
|
|
|
|
###############################################################################
|
2023-05-03 05:26:22 -05:00
|
|
|
START_TIME = echo Build start time: $$(date -ud "@$($@_TSTMP_ST)") $(LOG)
|
|
|
|
|
END_TIME = echo Build end time: $$(date -ud "@$($@_TSTMP_END)") $(LOG)
|
|
|
|
|
ELAPSED_TIME = echo Elapsed time: $$(date -ud "@$$(( $($@_TSTMP_END) - $($@_TSTMP_ST) ))" +'%-Hh %-Mm %-Ss') $(LOG)
|
2016-12-05 13:12:19 -06:00
|
|
|
|
|
|
|
|
# Dump targets taht current depends on
|
|
|
|
|
ifeq ($(SONIC_CONFIG_PRINT_DEPENDENCIES),y)
|
2017-07-25 01:49:39 -05:00
|
|
|
PRINT_DEPENDENCIES = echo Dependencies for $@ are $^ $(LOG)
|
2016-12-05 13:12:19 -06:00
|
|
|
endif
|
|
|
|
|
|
|
|
|
|
# header for each rule
|
|
|
|
|
define HEADER
|
2017-07-25 01:49:39 -05:00
|
|
|
@
|
2016-12-05 13:12:19 -06:00
|
|
|
$(PRINT_DEPENDENCIES)
|
|
|
|
|
$(FLUSH_LOG)
|
2023-05-03 05:26:22 -05:00
|
|
|
$(eval $@_TSTMP_ST := $(shell date +%s))
|
|
|
|
|
$(START_TIME)
|
2020-10-09 04:49:20 -05:00
|
|
|
./update_screen.sh -a $@ $*
|
2016-12-05 13:12:19 -06:00
|
|
|
endef
|
|
|
|
|
|
|
|
|
|
# footer for each rule
|
|
|
|
|
define FOOTER
|
2023-05-03 05:26:22 -05:00
|
|
|
$(eval $@_TSTMP_END := `date +%s`)
|
|
|
|
|
$(END_TIME)
|
|
|
|
|
$(ELAPSED_TIME)
|
2020-10-09 04:49:20 -05:00
|
|
|
./update_screen.sh -d $@ $($*_CACHE_LOADED)
|
2016-12-05 13:12:19 -06:00
|
|
|
endef
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Definition of derived target
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
# call:
|
|
|
|
|
# add_derived_package some_deb.deb, some_derived_deb
|
|
|
|
|
define add_derived_package
|
|
|
|
|
$(2)_DEPENDS += $(1)
|
|
|
|
|
$(2)_RDEPENDS += $($(1)_RDEPENDS)
|
2017-03-01 10:32:58 -06:00
|
|
|
$(2)_MAIN_DEB = $(1)
|
2016-12-05 13:12:19 -06:00
|
|
|
$(1)_DERIVED_DEBS += $(2)
|
|
|
|
|
$(2)_URL = $($(1)_URL)
|
|
|
|
|
$(2)_SRC_PATH = $($(1)_SRC_PATH)
|
2021-05-28 12:16:02 -05:00
|
|
|
$(2)_NAME = $($(1)_NAME)
|
|
|
|
|
$(2)_VERSION = $($(1)_VERSION)
|
2016-12-05 13:12:19 -06:00
|
|
|
SONIC_DERIVED_DEBS += $(2)
|
|
|
|
|
endef
|
|
|
|
|
|
2017-03-01 10:32:58 -06:00
|
|
|
# call:
|
|
|
|
|
# add_extra_package some_deb.deb, some_extra_deb
|
|
|
|
|
define add_extra_package
|
|
|
|
|
$(2)_MAIN_DEB = $(1)
|
|
|
|
|
$(1)_EXTRA_DEBS += $(2)
|
|
|
|
|
SONIC_EXTRA_DEBS += $(2)
|
|
|
|
|
endef
|
|
|
|
|
|
[build]: wait for conflicts package to be uninstalled (#5039)
when parallel build is enabled, both docker-fpm-frr and docker-syncd-brcm
is built at the same time, docker-fpm-frr requires swss which requires to
install libsaivs-dev. docker-syncd-brcm requires syncd package which requires
to install libsaibcm-dev.
since libsaivs-dev and libsaibcm-dev install the sai header in the same
location, these two packages cannot be installed at the same time. Therefore,
we need to serialize the build between these two packages. Simply uninstall
the conflict package is not enough to solve this issue. The correct solution
is to have one package wait for another package to be uninstalled.
For example, if syncd is built first, then it will install libsaibcm-dev.
Meanwhile, if the swss build job starts and tries to install libsaivs-dev,
it will first try to query if libsaibcm-dev is installed or not. if it is
installed, then it will wait until libsaibcm-dev is uninstalled. After syncd
job is finished, it will uninstall libsaibcm-dev and swss build job will be
unblocked.
To solve this issue, _UNINSTALLS is introduced to uninstall a package that
is no longer needed and to allow blocked job to continue.
Signed-off-by: Guohan Lu <lguohan@gmail.com>
2020-07-27 12:46:20 -05:00
|
|
|
###############################################################################
|
|
|
|
|
## Definition of conflict packages
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
# call:
|
|
|
|
|
# add_conflict_package some_deb.deb, conflict_deb
|
|
|
|
|
define add_conflict_package
|
|
|
|
|
$(1)_CONFLICT_DEBS += $(2)
|
|
|
|
|
$(2)_CONFLICT_DEBS += $(1)
|
|
|
|
|
endef
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Definition of debug dockers
|
|
|
|
|
###############################################################################
|
[build]: support for DPKG local caching (#4117)
DPKG caching framework provides the infrastructure to cache the sonic module/target .deb files into a local cache by tracking the target dependency files.SONIC build infrastructure is designed as a plugin framework where any new source code can be easily integrated into sonic as a module and that generates output as a .deb file. The source code compilation of a module is completely independent of other modules compilation. Inter module dependency is resolved through build artifacts like header files, libraries, and binaries in the form of Debian packages. For example module A depends on module B. While module A is being built, it uses B's .deb file to install it in the build docker.
The DPKG caching framework provides an infrastructure that caches a module's deb package and restores it back to the build directory if its dependency files are not modified. When a module is compiled for the first time, the generated deb package is stored at the DPKG cache location. On the subsequent build, first, it checks the module dependency file modification. If none of the dependent files is changed, it copies the deb package from the cache location, otherwise, it goes for local compilation and generates the deb package. The modified files should be checked-in to get the newer cache deb package.
This provides a huge improvement in build time and also supports the true incremental build by tracking the dependency files.
- How I did it
It takes two global arguments to enable the DPKG caching, the first one indicates the caching method and the second one describes the location of the cache.
SONIC_DPKG_CACHE_METHOD=cache
SONIC_DPKG_CACHE_SOURCE=
where SONIC_DPKG_CACHE_METHOD - Default method is 'cache' for deb package caching
none: no caching
cache: cache from local directory
Dependency file tracking:
Dependency files are tracked for each target in two levels.
1. Common make infrastructure files - rules/config, rules/functions, slave.mk etc.
2. Per module files - files which are specific to modules, Makefile, debian/rules, patch files, etc.
For example: dependency files for Linux Kernel - src/sonic-linux-kernel,
SPATH := $($(LINUX_HEADERS_COMMON)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/linux-kernel.mk rules/linux-kernel.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) \
$(KERNEL_PROCURE_METHOD) $(KERNEL_CACHE_PATH)
$(LINUX_HEADERS_COMMON)_CACHE_MODE := GIT_CONTENT_SHA
$(LINUX_HEADERS_COMMON)_DEP_FLAGS := $(DEP_FLAGS)
$(LINUX_HEADERS_COMMON)_DEP_FILES := $(DEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_FILES := $(SMDEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_PATHS := $(SPATH)
Cache file tracking:
The Cache file is a compressed TAR ball of a module's target DEB file and its derived-target DEB files.
The cache filename is formed with the following format
FORMAT:
<module deb filename>.<24 byte of DEP SHA hash >-<24 byte of MOD SHA hash>.tgz
Eg:
linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb-23658712fd21bb776fa16f47-c0b63ef593d4a32643bca228.tgz
< 24-byte DEP SHA value > - the SHA value is derived from all the dependent packages.
< 24-byte MOD SHA value > - the SHA value is derived from either of the following.
GIT_COMMIT_SHA - SHA value of the last git commit ID if it is a submodule
GIT_CONTENT_SHA - SHA value is generated from the content of the target dependency files.
Target Specific rules:
Caching can be enabled/disabled on a global level and also on the per-target level.
$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
$(call dpkg_depend,$(DEBS_PATH)/%.dep )
$(HEADER)
# Load the target deb from DPKG cache
$(call LOAD_CACHE,$*,$@)
# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then
.....
# Rules for Generating the target DEB file.
.....
# Save the target deb into DPKG cache
$(call SAVE_CACHE,$*,$@)
fi
$(FOOTER)
The make rule-'$(call dpkg_depend,$(DEBS_PATH)/%.dep )' checks for target dependency file modification. If it is newer than the target, it will go for re-generation of that target.
Two main macros 'LOAD_CACHE' and 'SAVE_CACHE' are used for loading and storing the cache contents.
The 'LOAD_CACHE' macro is used to load the cache file from cache storage and extracts them into the target folder. It is done only if target dependency files are not modified by checking the GIT file status, otherwise, cache loading is skipped and full compilation is performed.
It also updates the target-specific variable to indicate the cache is loaded or not.
The 'SAVE_CACHE' macro generates the compressed tarball of the cache file and saves them into cache storage. Saving into the cache storage is protected with a lock.
- How to verify it
The caching functionality is verified by enabling it in Linux kernel submodule.
It uses the cache directory as 'target/cache' where Linux cache file gets stored on the first-time build and it is picked from the cache location during the subsequent clean build.
- Description for the changelog
The DPKG caching framework provides the infrastructure to save the module-specific deb file to be cached by tracking the module's dependency files.
If the module's dependency files are not changed, it restores the module deb files from the cache storage.
- Description for the changelog
- A picture of a cute animal (not mandatory but encouraged)
DOCUMENT PR:
https://github.com/Azure/SONiC/pull/559
2020-03-11 22:04:52 -05:00
|
|
|
|
|
|
|
|
# call:
|
|
|
|
|
# add_dbg_docker some_docker.gz, some-docker-dbg.gz
|
|
|
|
|
define add_dbg_docker
|
|
|
|
|
$(2)_PATH = $($(1)_PATH)
|
|
|
|
|
$(2)_DBG_DEPENDS += $($(1)_DBG_DEPENDS)
|
|
|
|
|
$(2)_DBG_IMAGE_PACKAGES += $($(1)_DBG_IMAGE_PACKAGES)
|
2021-01-13 12:32:42 -06:00
|
|
|
$(2)_PYTHON_DEBS += $($(1)_PYTHON_DEBS)
|
[build]: support for DPKG local caching (#4117)
DPKG caching framework provides the infrastructure to cache the sonic module/target .deb files into a local cache by tracking the target dependency files.SONIC build infrastructure is designed as a plugin framework where any new source code can be easily integrated into sonic as a module and that generates output as a .deb file. The source code compilation of a module is completely independent of other modules compilation. Inter module dependency is resolved through build artifacts like header files, libraries, and binaries in the form of Debian packages. For example module A depends on module B. While module A is being built, it uses B's .deb file to install it in the build docker.
The DPKG caching framework provides an infrastructure that caches a module's deb package and restores it back to the build directory if its dependency files are not modified. When a module is compiled for the first time, the generated deb package is stored at the DPKG cache location. On the subsequent build, first, it checks the module dependency file modification. If none of the dependent files is changed, it copies the deb package from the cache location, otherwise, it goes for local compilation and generates the deb package. The modified files should be checked-in to get the newer cache deb package.
This provides a huge improvement in build time and also supports the true incremental build by tracking the dependency files.
- How I did it
It takes two global arguments to enable the DPKG caching, the first one indicates the caching method and the second one describes the location of the cache.
SONIC_DPKG_CACHE_METHOD=cache
SONIC_DPKG_CACHE_SOURCE=
where SONIC_DPKG_CACHE_METHOD - Default method is 'cache' for deb package caching
none: no caching
cache: cache from local directory
Dependency file tracking:
Dependency files are tracked for each target in two levels.
1. Common make infrastructure files - rules/config, rules/functions, slave.mk etc.
2. Per module files - files which are specific to modules, Makefile, debian/rules, patch files, etc.
For example: dependency files for Linux Kernel - src/sonic-linux-kernel,
SPATH := $($(LINUX_HEADERS_COMMON)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/linux-kernel.mk rules/linux-kernel.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) \
$(KERNEL_PROCURE_METHOD) $(KERNEL_CACHE_PATH)
$(LINUX_HEADERS_COMMON)_CACHE_MODE := GIT_CONTENT_SHA
$(LINUX_HEADERS_COMMON)_DEP_FLAGS := $(DEP_FLAGS)
$(LINUX_HEADERS_COMMON)_DEP_FILES := $(DEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_FILES := $(SMDEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_PATHS := $(SPATH)
Cache file tracking:
The Cache file is a compressed TAR ball of a module's target DEB file and its derived-target DEB files.
The cache filename is formed with the following format
FORMAT:
<module deb filename>.<24 byte of DEP SHA hash >-<24 byte of MOD SHA hash>.tgz
Eg:
linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb-23658712fd21bb776fa16f47-c0b63ef593d4a32643bca228.tgz
< 24-byte DEP SHA value > - the SHA value is derived from all the dependent packages.
< 24-byte MOD SHA value > - the SHA value is derived from either of the following.
GIT_COMMIT_SHA - SHA value of the last git commit ID if it is a submodule
GIT_CONTENT_SHA - SHA value is generated from the content of the target dependency files.
Target Specific rules:
Caching can be enabled/disabled on a global level and also on the per-target level.
$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
$(call dpkg_depend,$(DEBS_PATH)/%.dep )
$(HEADER)
# Load the target deb from DPKG cache
$(call LOAD_CACHE,$*,$@)
# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then
.....
# Rules for Generating the target DEB file.
.....
# Save the target deb into DPKG cache
$(call SAVE_CACHE,$*,$@)
fi
$(FOOTER)
The make rule-'$(call dpkg_depend,$(DEBS_PATH)/%.dep )' checks for target dependency file modification. If it is newer than the target, it will go for re-generation of that target.
Two main macros 'LOAD_CACHE' and 'SAVE_CACHE' are used for loading and storing the cache contents.
The 'LOAD_CACHE' macro is used to load the cache file from cache storage and extracts them into the target folder. It is done only if target dependency files are not modified by checking the GIT file status, otherwise, cache loading is skipped and full compilation is performed.
It also updates the target-specific variable to indicate the cache is loaded or not.
The 'SAVE_CACHE' macro generates the compressed tarball of the cache file and saves them into cache storage. Saving into the cache storage is protected with a lock.
- How to verify it
The caching functionality is verified by enabling it in Linux kernel submodule.
It uses the cache directory as 'target/cache' where Linux cache file gets stored on the first-time build and it is picked from the cache location during the subsequent clean build.
- Description for the changelog
The DPKG caching framework provides the infrastructure to save the module-specific deb file to be cached by tracking the module's dependency files.
If the module's dependency files are not changed, it restores the module deb files from the cache storage.
- Description for the changelog
- A picture of a cute animal (not mandatory but encouraged)
DOCUMENT PR:
https://github.com/Azure/SONiC/pull/559
2020-03-11 22:04:52 -05:00
|
|
|
$(2)_PYTHON_WHEELS += $($(1)_PYTHON_WHEELS)
|
|
|
|
|
$(2)_LOAD_DOCKERS += $($(1)_LOAD_DOCKERS)
|
|
|
|
|
$(2)_CACHE_MODE += $($(1)_CACHE_MODE)
|
|
|
|
|
$(2)_DEP_FLAGS += $($(1)_DEP_FLAGS)
|
|
|
|
|
$(2)_DEP_FILES += $($(1)_DEP_FILES)
|
|
|
|
|
$(2)_SMDEP_FILES += $($(1)_SMDEP_FILES)
|
|
|
|
|
$(2)_SMDEP_PATHS += $($(1)_SMDEP_PATHS)
|
|
|
|
|
$(2)_DEPENDS += $($(1)_DEPENDS)
|
|
|
|
|
$(2)_RDEPENDS += $($(1)_RDEPENDS)
|
|
|
|
|
$(2)_MAIN_DOCKER = $(1)
|
|
|
|
|
$(1)_DBG_DOCKER += $(2)
|
|
|
|
|
SONIC_DBG_DOCKERS += $(2)
|
|
|
|
|
endef
|
|
|
|
|
|
|
|
|
|
|
2016-12-05 13:12:19 -06:00
|
|
|
###############################################################################
|
|
|
|
|
## Utility functions
|
|
|
|
|
###############################################################################
|
[build]: support for DPKG local caching (#4117)
DPKG caching framework provides the infrastructure to cache the sonic module/target .deb files into a local cache by tracking the target dependency files.SONIC build infrastructure is designed as a plugin framework where any new source code can be easily integrated into sonic as a module and that generates output as a .deb file. The source code compilation of a module is completely independent of other modules compilation. Inter module dependency is resolved through build artifacts like header files, libraries, and binaries in the form of Debian packages. For example module A depends on module B. While module A is being built, it uses B's .deb file to install it in the build docker.
The DPKG caching framework provides an infrastructure that caches a module's deb package and restores it back to the build directory if its dependency files are not modified. When a module is compiled for the first time, the generated deb package is stored at the DPKG cache location. On the subsequent build, first, it checks the module dependency file modification. If none of the dependent files is changed, it copies the deb package from the cache location, otherwise, it goes for local compilation and generates the deb package. The modified files should be checked-in to get the newer cache deb package.
This provides a huge improvement in build time and also supports the true incremental build by tracking the dependency files.
- How I did it
It takes two global arguments to enable the DPKG caching, the first one indicates the caching method and the second one describes the location of the cache.
SONIC_DPKG_CACHE_METHOD=cache
SONIC_DPKG_CACHE_SOURCE=
where SONIC_DPKG_CACHE_METHOD - Default method is 'cache' for deb package caching
none: no caching
cache: cache from local directory
Dependency file tracking:
Dependency files are tracked for each target in two levels.
1. Common make infrastructure files - rules/config, rules/functions, slave.mk etc.
2. Per module files - files which are specific to modules, Makefile, debian/rules, patch files, etc.
For example: dependency files for Linux Kernel - src/sonic-linux-kernel,
SPATH := $($(LINUX_HEADERS_COMMON)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/linux-kernel.mk rules/linux-kernel.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) \
$(KERNEL_PROCURE_METHOD) $(KERNEL_CACHE_PATH)
$(LINUX_HEADERS_COMMON)_CACHE_MODE := GIT_CONTENT_SHA
$(LINUX_HEADERS_COMMON)_DEP_FLAGS := $(DEP_FLAGS)
$(LINUX_HEADERS_COMMON)_DEP_FILES := $(DEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_FILES := $(SMDEP_FILES)
$(LINUX_HEADERS_COMMON)_SMDEP_PATHS := $(SPATH)
Cache file tracking:
The Cache file is a compressed TAR ball of a module's target DEB file and its derived-target DEB files.
The cache filename is formed with the following format
FORMAT:
<module deb filename>.<24 byte of DEP SHA hash >-<24 byte of MOD SHA hash>.tgz
Eg:
linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb-23658712fd21bb776fa16f47-c0b63ef593d4a32643bca228.tgz
< 24-byte DEP SHA value > - the SHA value is derived from all the dependent packages.
< 24-byte MOD SHA value > - the SHA value is derived from either of the following.
GIT_COMMIT_SHA - SHA value of the last git commit ID if it is a submodule
GIT_CONTENT_SHA - SHA value is generated from the content of the target dependency files.
Target Specific rules:
Caching can be enabled/disabled on a global level and also on the per-target level.
$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
$(call dpkg_depend,$(DEBS_PATH)/%.dep )
$(HEADER)
# Load the target deb from DPKG cache
$(call LOAD_CACHE,$*,$@)
# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then
.....
# Rules for Generating the target DEB file.
.....
# Save the target deb into DPKG cache
$(call SAVE_CACHE,$*,$@)
fi
$(FOOTER)
The make rule-'$(call dpkg_depend,$(DEBS_PATH)/%.dep )' checks for target dependency file modification. If it is newer than the target, it will go for re-generation of that target.
Two main macros 'LOAD_CACHE' and 'SAVE_CACHE' are used for loading and storing the cache contents.
The 'LOAD_CACHE' macro is used to load the cache file from cache storage and extracts them into the target folder. It is done only if target dependency files are not modified by checking the GIT file status, otherwise, cache loading is skipped and full compilation is performed.
It also updates the target-specific variable to indicate the cache is loaded or not.
The 'SAVE_CACHE' macro generates the compressed tarball of the cache file and saves them into cache storage. Saving into the cache storage is protected with a lock.
- How to verify it
The caching functionality is verified by enabling it in Linux kernel submodule.
It uses the cache directory as 'target/cache' where Linux cache file gets stored on the first-time build and it is picked from the cache location during the subsequent clean build.
- Description for the changelog
The DPKG caching framework provides the infrastructure to save the module-specific deb file to be cached by tracking the module's dependency files.
If the module's dependency files are not changed, it restores the module deb files from the cache storage.
- Description for the changelog
- A picture of a cute animal (not mandatory but encouraged)
DOCUMENT PR:
https://github.com/Azure/SONiC/pull/559
2020-03-11 22:04:52 -05:00
|
|
|
# Recursive wildcard function
|
|
|
|
|
rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d))
|
2016-12-05 13:12:19 -06:00
|
|
|
|
|
|
|
|
expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1)
|
[build]: wait for conflicts package to be uninstalled (#5039)
when parallel build is enabled, both docker-fpm-frr and docker-syncd-brcm
is built at the same time, docker-fpm-frr requires swss which requires to
install libsaivs-dev. docker-syncd-brcm requires syncd package which requires
to install libsaibcm-dev.
since libsaivs-dev and libsaibcm-dev install the sai header in the same
location, these two packages cannot be installed at the same time. Therefore,
we need to serialize the build between these two packages. Simply uninstall
the conflict package is not enough to solve this issue. The correct solution
is to have one package wait for another package to be uninstalled.
For example, if syncd is built first, then it will install libsaibcm-dev.
Meanwhile, if the swss build job starts and tries to install libsaivs-dev,
it will first try to query if libsaibcm-dev is installed or not. if it is
installed, then it will wait until libsaibcm-dev is uninstalled. After syncd
job is finished, it will uninstall libsaibcm-dev and swss build job will be
unblocked.
To solve this issue, _UNINSTALLS is introduced to uninstall a package that
is no longer needed and to allow blocked job to continue.
Signed-off-by: Guohan Lu <lguohan@gmail.com>
2020-07-27 12:46:20 -05:00
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Uninstall debs
|
|
|
|
|
###############################################################################
|
|
|
|
|
define UNINSTALL_DEBS
|
2021-01-01 03:00:47 -06:00
|
|
|
if [ -n "$(1)" ]; then \
|
|
|
|
|
while true; do \
|
|
|
|
|
if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then \
|
|
|
|
|
{ sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(foreach deb,$(1),$(firstword $(subst _, ,$(basename $(deb))))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } \
|
|
|
|
|
fi; \
|
|
|
|
|
done; \
|
|
|
|
|
fi
|
[build]: wait for conflicts package to be uninstalled (#5039)
when parallel build is enabled, both docker-fpm-frr and docker-syncd-brcm
is built at the same time, docker-fpm-frr requires swss which requires to
install libsaivs-dev. docker-syncd-brcm requires syncd package which requires
to install libsaibcm-dev.
since libsaivs-dev and libsaibcm-dev install the sai header in the same
location, these two packages cannot be installed at the same time. Therefore,
we need to serialize the build between these two packages. Simply uninstall
the conflict package is not enough to solve this issue. The correct solution
is to have one package wait for another package to be uninstalled.
For example, if syncd is built first, then it will install libsaibcm-dev.
Meanwhile, if the swss build job starts and tries to install libsaivs-dev,
it will first try to query if libsaibcm-dev is installed or not. if it is
installed, then it will wait until libsaibcm-dev is uninstalled. After syncd
job is finished, it will uninstall libsaibcm-dev and swss build job will be
unblocked.
To solve this issue, _UNINSTALLS is introduced to uninstall a package that
is no longer needed and to allow blocked job to continue.
Signed-off-by: Guohan Lu <lguohan@gmail.com>
2020-07-27 12:46:20 -05:00
|
|
|
endef
|
2021-01-12 08:03:12 -06:00
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Setup overlay fs for dpkg admin directory /var/lib/dpkg
|
|
|
|
|
###############################################################################
|
|
|
|
|
define SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR
|
|
|
|
|
upperdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
|
|
|
|
|
workdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
|
|
|
|
|
mergedir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
|
|
|
|
|
sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workdir=$$workdir $$mergedir
|
|
|
|
|
export SONIC_DPKG_ADMINDIR=$$mergedir
|
|
|
|
|
trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT
|
|
|
|
|
endef
|
2021-04-26 15:51:50 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
## Manifest generation routine
|
|
|
|
|
## TODO: changelog generation
|
|
|
|
|
###############################################################################
|
|
|
|
|
# call:
|
|
|
|
|
# generate_manifest some_docker.gz, version_suffix
|
|
|
|
|
define generate_manifest
|
2021-11-01 12:02:57 -05:00
|
|
|
$(eval export version=$($(1).gz_VERSION))
|
|
|
|
|
$(eval export version_suffix=$(2))
|
2021-04-26 15:51:50 -05:00
|
|
|
$(eval export name=$($(1).gz_CONTAINER_NAME))
|
|
|
|
|
$(eval export package_name=$($(1).gz_PACKAGE_NAME))
|
|
|
|
|
$(eval export asic_service=$(shell [ -f files/build_templates/per_namespace/$(name).service.j2 ] && echo true || echo false))
|
|
|
|
|
$(eval export host_service=$(shell [ -f files/build_templates/$(name).service.j2 ] && echo true || echo false))
|
|
|
|
|
$(eval export depends=$($(1).gz_PACKAGE_DEPENDS))
|
|
|
|
|
$(eval export requires=$($(1).gz_SERVICE_REQUIRES))
|
|
|
|
|
$(eval export after=$($(1).gz_SERVICE_AFTER))
|
|
|
|
|
$(eval export before=$($(1).gz_SERVICE_BEFORE))
|
|
|
|
|
$(eval export dependent_of=$($(1).gz_SERVICE_DEPENDENT_OF))
|
2021-07-11 08:58:05 -05:00
|
|
|
$(eval export warm_shutdown_after=$($*.gz_WARM_SHUTDOWN_AFTER))
|
|
|
|
|
$(eval export warm_shutdown_before=$($*.gz_WARM_SHUTDOWN_BEFORE))
|
|
|
|
|
$(eval export fast_shutdown_after=$($*.gz_FAST_SHUTDOWN_AFTER))
|
|
|
|
|
$(eval export fast_shutdown_before=$($*.gz_FAST_SHUTDOWN_BEFORE))
|
2021-04-26 15:51:50 -05:00
|
|
|
$(eval export privileged=$($(1).gz_CONTAINER_PRIVILEGED))
|
|
|
|
|
$(eval export volumes=$($(1).gz_CONTAINER_VOLUMES))
|
|
|
|
|
$(eval export tmpfs=$($(1).gz_CONTAINER_TMPFS))
|
|
|
|
|
$(eval export config_cli_plugin=$($(1).gz_CLI_CONFIG_PLUGIN))
|
|
|
|
|
$(eval export show_cli_plugin=$($(1).gz_CLI_SHOW_PLUGIN))
|
2021-09-24 00:01:26 -05:00
|
|
|
$(eval export clear_cli_plugin=$($(1).gz_CLI_CLEAR_PLUGIN))
|
2022-12-20 02:53:58 -06:00
|
|
|
$(eval export support_rate_limit=$($(1).gz_SUPPORT_RATE_LIMIT))
|
2021-04-26 15:51:50 -05:00
|
|
|
j2 --customize scripts/j2cli/json_filter.py files/build_templates/manifest.json.j2 > $($(1).gz_PATH)/manifest.common.json
|
|
|
|
|
if [ -f $($*.gz_PATH)/manifest.part.json.j2 ]; then
|
|
|
|
|
j2 --customize scripts/j2cli/json_filter.py $($(1).gz_PATH)/manifest.part.json.j2 > $($(1).gz_PATH)/manifest.part.json
|
|
|
|
|
jq -s '.[0] * .[1]' $($(1).gz_PATH)/manifest.common.json $($(1).gz_PATH)/manifest.part.json > $($(1).gz_PATH)/manifest.json
|
|
|
|
|
else
|
|
|
|
|
mv $($(1).gz_PATH)/manifest.common.json $($(1).gz_PATH)/manifest.json
|
|
|
|
|
fi
|
|
|
|
|
endef
|
