54de99ba23
* [build] Add version files to docker image dependencies * [ci] Support to skip vstest using include/exclude config file. (#11086) example: ├── folderA │ ├── fileA (skip vstest) │ ├── fileB │ └── fileC If we want to skip vstest when changing /folderA/fileA, and not skip vstest when changing fileB or fileC. vstest-include: ^folderA/fileA vstest-exclude: ^folderA
693 lines
32 KiB
Makefile
693 lines
32 KiB
Makefile
#######################################################################
|
||
#
|
||
# Copyright (c) 2020 Broadcom, Inc.
|
||
# The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
|
||
#
|
||
# Author: Kalimuthu Velappan
|
||
# Greg Paussa
|
||
#
|
||
# Email : kalimuthu.velappan@broadcom.com
|
||
# greg.paussa@broadcom.com
|
||
#
|
||
# This program is free software; you can redistribute it and/or modify
|
||
# it under the terms of the GNU General Public License as published by
|
||
# the Free Software Foundation; either version 2 of the License, or
|
||
# (at your option) any later version.
|
||
#
|
||
# This program is distributed in the hope that it will be useful,
|
||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
# GNU General Public License for more details.
|
||
#
|
||
#######################################################################
|
||
#
|
||
#
|
||
# DPKG caching framework
|
||
#
|
||
# SONiC source code composes of multiple open source modules. Eg: Linux, bash, ntp etc.
|
||
# Each module considered as a build target in the sonic build framework.
|
||
# Each module gets compiled and generate the final debian package(.deb) file.
|
||
# There are two types of source code packages used by the SONiC repo.
|
||
#
|
||
# 1. Module source code is maintained as part of main repo
|
||
# Eg: sonic-utilities, Plaform files etc
|
||
#
|
||
# Some module source code is maintained outside the sonic repo, but
|
||
# the build framework is part of sonic main repo.
|
||
# The build framework downloads the zipped source content from the web,
|
||
# applies a series of patches (if applicable) on the downloaded source code,
|
||
# compiles the source and generates the final .deb package(s).
|
||
# Eg: bash, ntp, etc
|
||
#
|
||
# 2. Module source code is maintained as a submodule(SM) in the sonic repo.
|
||
# Eg: Frr, swss, Linux etc
|
||
#
|
||
#
|
||
# The sonic build framework uses the module .deb packages that are generated as part of build and
|
||
# creates the set of target docker images and the final binary image for distribution.
|
||
#
|
||
# The caching framework provides the method to cache the module .deb packages and docker images
|
||
# into a cache location by tracking module dependency information.
|
||
#
|
||
# A module can have a set of dependency files, for example Makefiles, source files, scripts, dpkg control files, etc.
|
||
# The caching is done based on the SHA hash value of the module dependency file contents. If one of the
|
||
# dependency files is changed, the corresponding cache file is also changed. So it automatically creates the new cache file
|
||
# for that module.
|
||
#
|
||
# It provides two levels of caching.
|
||
# Global cache := Module doesnt have any local changes.
|
||
# Local cache := Used for target when one of its dependency file is modified.
|
||
#
|
||
#
|
||
# Steps for adding new module is given as a template.
|
||
# Template File : rules/template.dep
|
||
#
|
||
|
||
|
||
# Common files and FLAGS
|
||
# Run the 'touch cache.skip.common' command in the base directory to exclude the common files from caching
|
||
SONIC_COMMON_FILES_LIST := $(if $(wildcard cache.skip.common),, .platform slave.mk rules/functions Makefile.cache)
|
||
SONIC_COMMON_FLAGS_LIST := $(CONFIGURED_PLATFORM) \
|
||
$(SONIC_DEBUGGING_ON) \
|
||
$(SONIC_PROFILING_ON) $(SONIC_ENABLE_SYNCD_RPC)
|
||
SONIC_COMMON_DPKG_LIST := debian/control debian/changelog debian/rules \
|
||
debian/compat debian/install debian/copyright
|
||
SONIC_COMMON_BASE_FILES_LIST := sonic-slave-jessie/Dockerfile.j2 sonic-slave-jessie/Dockerfile.user.j2 \
|
||
sonic-slave-stretch/Dockerfile.j2 sonic-slave-stretch/Dockerfile.user.j2 \
|
||
sonic-slave-buster/Dockerfile.j2 sonic-slave-buster/Dockerfile.user.j2
|
||
|
||
|
||
|
||
include $(RULES_PATH)/*.dep
|
||
|
||
|
||
ifneq ($(CONFIGURED_PLATFORM), undefined)
|
||
ifeq ($(PDDF_SUPPORT), y)
|
||
include $(PLATFORM_PDDF_PATH)/rules.dep
|
||
endif
|
||
|
||
-include $(PLATFORM_PATH)/rules.dep
|
||
endif
|
||
|
||
ifndef SONIC_BUILD_QUIETER
|
||
$(info "SONIC_DPKG_CACHE_METHOD" : "$(SONIC_DPKG_CACHE_METHOD)")
|
||
ifneq ($(SONIC_DPKG_CACHE_METHOD),none)
|
||
$(info "DPKG_CACHE_PATH" : "$(SONIC_DPKG_CACHE_SOURCE)")
|
||
endif
|
||
$(info )
|
||
endif
|
||
|
||
|
||
###############################################################################
|
||
## Canned sequences
|
||
###############################################################################
|
||
|
||
SONIC_DPKG_CACHE_DIR := /dpkg_cache
|
||
MOD_CACHE_LOCK_SUFFIX := cache_access
|
||
MOD_CACHE_LOCK_TIMEOUT := 3600
|
||
SONIC_DPKG_LOCAL_CACHE_DIR=${TARGET_PATH}/cache
|
||
$(shell test -d $(SONIC_DPKG_LOCAL_CACHE_DIR) || \
|
||
mkdir -p $(SONIC_DPKG_LOCAL_CACHE_DIR) && chmod 777 $(SONIC_DPKG_LOCAL_CACHE_DIR) )
|
||
$(shell test -w $(SONIC_DPKG_CACHE_DIR) || sudo chmod 777 $(SONIC_DPKG_CACHE_DIR) )
|
||
|
||
DOCKER_LOCKFILE_SUFFIX := access
|
||
DOCKER_LOCKFILE_TIMEOUT := 1200
|
||
|
||
# Lock macro for shared file access
|
||
# Lock is implemented through flock command with a specified timeout value
|
||
# Lock file is created in the specified directory, a separate one for each target file name
|
||
# A designated suffix is appended to each target file name, followed by .lock
|
||
#
|
||
# Parameters:
|
||
# $(1) - target file name (without path)
|
||
# $(2) - lock file path (only)
|
||
# $(3) - designated lock file suffix
|
||
# $(4) - flock timeout (in seconds)
|
||
#
|
||
# $(call MOD_LOCK,file,path,suffix,timeout)
|
||
define MOD_LOCK
|
||
if [[ ! -f $(2)/$(1)_$(3).lock ]]; then
|
||
touch $(2)/$(1)_$(3).lock
|
||
chmod 777 $(2)/$(1)_$(3).lock;
|
||
fi
|
||
$(eval $(1)_lock_fd=$(subst ~,_,$(subst -,_,$(subst +,_,$(subst .,_,$(1))))))
|
||
exec {$($(1)_lock_fd)}<>"$(2)/$(1)_$(3).lock";
|
||
if ! flock -x -w $(4) "$${$($(1)_lock_fd)}" ; then
|
||
echo "ERROR: Lock timeout trying to access $(2)/$(1)_$(3).lock";
|
||
exit 1;
|
||
fi
|
||
endef
|
||
|
||
# UnLock macro for shared file access
|
||
#
|
||
# Parameters:
|
||
# $(1) - target file name (without path)
|
||
#
|
||
# $(call MOD_UNLOCK,file)
|
||
define MOD_UNLOCK
|
||
eval exec "$${$($(1)_lock_fd)}<&-";
|
||
endef
|
||
|
||
|
||
# Calculate the 24 byte SHA value
|
||
# GIT_COMMIT_SHA => SHA is derived from last git commit ID
|
||
# GIT_CONTENT_SHA => SHA is derived from contents of depdency files
|
||
# Args:
|
||
# $(1) => target name
|
||
define GET_MOD_SHA
|
||
$(eval $(1)_MOD_DEP_FILES := $($(1)_DEP_FLAGS_FILE) $($(1)_MOD_HASH_FILE) $($(1)_SMOD_HASH_FILE) )
|
||
$(if $(MDEBUG), $(info $(1)_MOD_DEP_FILES: $($(1)_MOD_DEP_FILES)))
|
||
$(eval $(1)_MOD_HASH := $(if $(filter GIT_COMMIT_SHA,$($(1)_CACHE_MODE)),\
|
||
$(shell cd $($(1)_MOD_SRC_PATH) && git log -1 --format="%H"| awk '{print substr($$1,0,23);}' ),\
|
||
$(shell git hash-object $($(1)_MOD_DEP_FILES)| \
|
||
sha1sum | awk '{print substr($$1,0,23);}')))
|
||
endef
|
||
|
||
|
||
# Calculate the 24 byte SHA value
|
||
# SHA value is derived from dependent files of the target which includes .flags, .sha and .smsha files.
|
||
# Args:
|
||
# $(1) => target name
|
||
define GET_MOD_DEP_SHA
|
||
$(eval $(1)_MOD_DEP_PKGS := $(foreach dfile,$($(1)_DEPENDS) $($(1)_RDEPENDS) $($(1)_WHEEL_DEPENDS) \
|
||
$($(1)_PYTHON_DEBS) $($(1)_PYTHON_WHEELS) \
|
||
$($(1)_DBG_DEPENDS) $($(1)_DBG_IMAGE_PACKAGES) $($(1)_LOAD_DOCKERS),\
|
||
$(if $($(dfile)_MAIN_DEB),$($(dfile)_MAIN_DEB),$(dfile))) )
|
||
|
||
$(if $(MDEBUG), $(info $(1)_MOD_DEP_PKGS: $($(1)_MOD_DEP_PKGS)))
|
||
|
||
# Warn if there is any missing dependency files
|
||
$(eval $(1)_DEP_MOD_SHA_FILES := $(foreach dfile,$($(1)_MOD_DEP_PKGS), \
|
||
$($(dfile)_DEP_FLAGS_FILE) $($(dfile)_MOD_HASH_FILE) $($(dfile)_SMOD_HASH_FILE)) )
|
||
$(eval $(1)_DEP_FILES_MISSING := $(filter-out $(wildcard $($(1)_DEP_MOD_SHA_FILES)),$($(1)_DEP_MOD_SHA_FILES)) )
|
||
$(if $($(1)_DEP_FILES_MISSING), $(warning "[ DPKG ] Dependecy file(s) are not found for $(1) : $($(1)_DEP_FILES_MISSING)))
|
||
|
||
# Include package dependencies hash values into package hash calculation
|
||
$(eval $(1)_DEP_PKGS_SHA := $(foreach dfile,$($(1)_MOD_DEP_PKGS),$($(dfile)_DEP_MOD_SHA) $($(dfile)_MOD_HASH)))
|
||
|
||
$(eval $(1)_DEP_MOD_SHA := $(shell bash -c "git hash-object $($(1)_DEP_MOD_SHA_FILES) && echo $($(1)_DEP_PKGS_SHA)" \
|
||
| sha1sum | awk '{print substr($$1,0,23);}'))
|
||
endef
|
||
|
||
|
||
# Retrive the list of files that are modified for the target. The files can be from
|
||
# 1. Any of dependent target is modified
|
||
# 2. Files from the target dependency list
|
||
# 3. Files from submodule dependency list if the target is a submodule
|
||
#
|
||
# Args:
|
||
# $(1) => target name
|
||
define GET_MODIFIED_FILES
|
||
$(eval $(1)_FILES_MODIFIED := $(foreach dfile,$($(1)_MOD_DEP_PKGS),$(if $($(dfile)_FILES_MODIFIED),$(dfile))) \
|
||
$(if $($(1)_DEP_FILES), $(shell cat $($(1)_MOD_DEP_FILE) | xargs git status -s)) \
|
||
$(if $($(1)_SMDEP_PATHS), $(foreach path,$($(1)_SMDEP_PATHS), \
|
||
$(shell cd $(path) && git status --ignore-submodules=all -s -uno .))) )
|
||
|
||
endef
|
||
|
||
# Loads the deb package from debian cache
|
||
# Cache file prefix is formed using SHA value
|
||
# The SHA value consists of
|
||
# 1. 24 byte SHA value is derived from the dependency files list
|
||
# Flags: Module ENV flags file
|
||
# Dependent packages : DEPENDS, RDEPENDS, WHEEL_DEPENDS, PYTHON_DEBS, PYTHON_WHEELS, DBG_DEPENDS, DBG_IMAGE_PACKAGES, LOAD_DOCKERS
|
||
# 2. 24 byte SHA value from one of the keyword type - GIT_COMMIT_SHA or GIT_CONTENT_SHA
|
||
# GIT_COMMIT_SHA - SHA value of the last git commit id if it is a submodule
|
||
# GIT_CONTENT_SHA - SHA value is calculated from the target dependency files content.
|
||
# Cache is loaded from either local cache or global cache based on the dependency SHA match with the cache filename.
|
||
# Otherwise it builds the package from source.
|
||
# TODO:
|
||
# 1. Extend dpkg for all the chip vendor packages.
|
||
|
||
# Args:
|
||
# $(1) => target name
|
||
# $(2) => target output file name
|
||
define LOAD_FROM_CACHE
|
||
|
||
# Calculate the modules SHA and its dependency SHA value
|
||
$(call GET_MOD_DEP_SHA,$(1))
|
||
$(call GET_MOD_SHA,$(1))
|
||
|
||
# Form the cache file name
|
||
$(eval $(1)_MOD_CACHE_FILE := $(1)-$($(1)_DEP_MOD_SHA)-$($(1)_MOD_HASH).tgz)
|
||
$(if $(MDEBUG), $(info $(1)_MODE_CACHE_FILE := $($(1)_MOD_CACHE_FILE)))
|
||
|
||
# Retrive and log files list that are modified for the target.
|
||
$(call GET_MODIFIED_FILES,$(1))
|
||
$(if $($(1)_FILES_MODIFIED),
|
||
echo "Target $(1) dependencies are modifed - global cache skipped" >> $($(1)_DST_PATH)/$(1).log
|
||
echo "Modified dependencies are : [$($(1)_FILES_MODIFIED)] " >> $($(1)_DST_PATH)/$(1).log
|
||
$(eval $(1)_CACHE_DIR := $(SONIC_DPKG_LOCAL_CACHE_DIR)))
|
||
|
||
# Choose the cache file path in the following order
|
||
# 1. First load from Local cache path
|
||
# 2. If not, load from global cache path
|
||
$(eval CACHE_FILE_SELECT:=$(or $(wildcard $(SONIC_DPKG_LOCAL_CACHE_DIR)/$($(1)_MOD_CACHE_FILE)), \
|
||
$(wildcard $(SONIC_DPKG_CACHE_DIR)/$($(1)_MOD_CACHE_FILE))) )
|
||
|
||
# Check if any of the derived package is not built
|
||
$(eval LOAD_DRV_DEB := $(foreach pkg,$(addprefix $($(1)_DST_PATH)/,$(1) $($(1)_DERIVED_DEBS) $($(1)_EXTRA_DEBS)),$(if $(wildcard $(pkg)),,$(pkg))))
|
||
|
||
# Load the cache if cache is enabled and cache file is present in the cache
|
||
# Update the cache_loaded variable
|
||
$(if $(and $(CACHE_FILE_SELECT),$(filter $(RCACHE_OPTIONS),$(SONIC_DPKG_CACHE_METHOD))),
|
||
$(if $(LOAD_DRV_DEB), $($(1)_CACHE_USER) tar -C $($(1)_BASE_PATH) -mxzvf $(CACHE_FILE_SELECT) 1>> $($(1)_DST_PATH)/$(1).log ,echo );
|
||
echo "File $(CACHE_FILE_SELECT) is loaded from cache into $($(1)_BASE_PATH)" >> $($(1)_DST_PATH)/$(1).log
|
||
$(eval $(1)_CACHE_LOADED := Yes)
|
||
$(if $(call CHECK_WCACHE_ENABLED,$(1)), $(shell touch $(CACHE_FILE_SELECT)))
|
||
echo "[ CACHE::LOADED ] $($(1)_CACHE_DIR)/$($(1)_MOD_CACHE_FILE)" >> $($(1)_DST_PATH)/$(1).log
|
||
echo "File $($(1)_CACHE_DIR)/$($(1)_MOD_CACHE_FILE) is not present in cache or cache mode set as $(SONIC_DPKG_CACHE_METHOD) !" >> $($(1)_DST_PATH)/$(1).log
|
||
echo "[ CACHE::SKIPPED ] $($(1)_CACHE_DIR)/$($(1)_MOD_CACHE_FILE)" >> $($(1)_DST_PATH)/$(1).log
|
||
echo "[ CACHE::SKIPPED ] DEP_FILES - Modified Files: [$($(1)_FILES_MODIFIED)] " >> $($(1)_DST_PATH)/$(1).log
|
||
echo "[ CACHE::SKIPPED ] DEPENDS - Modified Files: [$?] " >> $($(1)_DST_PATH)/$(1).log
|
||
)
|
||
endef
|
||
|
||
# Saves the deb package into debian cache
|
||
# A single tared-zip cache is created for .deb and its derived packages in the cache direcory.
|
||
# It saves the .deb into global cache only when its dependencies are not changed,
|
||
# Otherwise it saves the .deb into local cache
|
||
# The cache save is protected with lock.
|
||
# Args:
|
||
# $(1) => target name
|
||
# $(2) => target output file name
|
||
define SAVE_INTO_CACHE
|
||
|
||
# Calculate the modules SHA and its dependency SHA value
|
||
$(call GET_MOD_DEP_SHA,$(1))
|
||
$(call GET_MOD_SHA,$(1))
|
||
|
||
# Form the cache file name
|
||
$(eval $(1)_MOD_CACHE_FILE := $(1)-$($(1)_DEP_MOD_SHA)-$($(1)_MOD_HASH).tgz)
|
||
$(if $(MDEBUG), $(info $(1)_MOD_CACHE_FILE := $($(1)_MOD_CACHE_FILE)))
|
||
|
||
# Retrive and log files list that are modified for the target.
|
||
$(call GET_MODIFIED_FILES,$(1))
|
||
|
||
$(eval MOD_CACHE_FILE=$($(1)_MOD_CACHE_FILE))
|
||
$(call MOD_LOCK,$(1),$(SONIC_DPKG_CACHE_DIR),$(MOD_CACHE_LOCK_SUFFIX),$(MOD_CACHE_LOCK_TIMEOUT))
|
||
$(if $($(1)_FILES_MODIFIED),
|
||
echo "Target $(1) dependencies are modifed - global save cache skipped" >> $($(1)_DST_PATH)/$(1).log
|
||
$(eval $(1)_CACHE_DIR := $(SONIC_DPKG_LOCAL_CACHE_DIR))
|
||
)
|
||
$($(1)_CACHE_USER) tar -C $($(1)_BASE_PATH) -mczvf $($(1)_CACHE_DIR)/$(MOD_CACHE_FILE) $(2) $(addprefix $($(1)_DST_PATH)/,$($(1)_DERIVED_DEBS) $($(1)_EXTRA_DEBS) ) \
|
||
1>>$($(1)_DST_PATH)/$(1).log
|
||
sudo chmod 777 $($(1)_CACHE_DIR)/$(MOD_CACHE_FILE)
|
||
|
||
echo "File $($(1)_CACHE_DIR)/$(MOD_CACHE_FILE) saved in cache " >> $($(1)_DST_PATH)/$(1).log
|
||
echo "[ CACHE::SAVED ] $($(1)_CACHE_DIR)/$(MOD_CACHE_FILE)" >> $($(1)_DST_PATH)/$(1).log
|
||
|
||
$(call MOD_UNLOCK,$(1))
|
||
endef
|
||
|
||
# Read from the cache
|
||
RCACHE_OPTIONS := cache rcache rwcache
|
||
define CHECK_RCACHE_ENABLED
|
||
$(if $(and $(filter $(RCACHE_OPTIONS),$(SONIC_DPKG_CACHE_METHOD)),$(filter-out none,$($(1)_CACHE_MODE))),enabled)
|
||
endef
|
||
|
||
# Write into the cache
|
||
WCACHE_OPTIONS := cache wcache rwcache
|
||
define CHECK_WCACHE_ENABLED
|
||
$(if $(and $(filter $(WCACHE_OPTIONS),$(SONIC_DPKG_CACHE_METHOD)),$(filter-out none,$($(1)_CACHE_MODE))),enabled)
|
||
endef
|
||
|
||
# It logs the reason why the target is getting built/rebuilt
|
||
# Args:
|
||
# $(1) => target name
|
||
define SHOW_WHY
|
||
@echo "[ REASON ] :\
|
||
$(if $(filter $(PHONY),$@), it is phony,\
|
||
$(eval $(1)_PREREQ_PHONY:= $(filter $(PHONY),$^))\
|
||
$(eval $(1)_PREREQ_DNE:= $(filter-out $(wildcard $^) $($(1)_PREREQ_PHONY),$^))\
|
||
$(eval $(1)_PREREQ_NEW:= $(filter-out $($(1)_PREREQ_DNE),$?))\
|
||
$(if $(wildcard $@),$(if $($(1)_PREREQ_NEW), NEWER PREREQUISITES: $($(1)_PREREQ_NEW)), $@ does not exist)\
|
||
$(if $($(1)_PREREQ_DNE), NON-EXISTENT PREREQUISITES: $($(1)_PREREQ_DNE))\
|
||
$(if $($(1)_PREREQ_PHONY), PHONY PREREQUISITES: $($(1)_PREREQ_PHONY)))" >> $($(1)_DST_PATH)/$(1).log
|
||
|
||
@echo "[ FLAGS FILE ] : [$($(1)_FILE_FLAGS)] " >> $($(1)_DST_PATH)/$(1).log
|
||
@echo "[ FLAGS DEPENDS ] : [$($(1)_DEP_FLAGS_ALL)] " >> $($(1)_DST_PATH)/$(1).log
|
||
@echo "[ FLAGS DIFF ] : [$($(1)_FLAGS_DIFF)] " >> $($(1)_DST_PATH)/$(1).log
|
||
@$(file >>$($(1)_DST_PATH)/$(1).log, "[ DEP DEPENDS ] : [$($(1)_DEP_FILES_MODIFIED)] ")
|
||
@$(file >>$($(1)_DST_PATH)/$(1).log, "[ SMDEP DEPENDS ] : [$($(1)_SMDEP_FILES_MODIFIED)] ")
|
||
@$(file >>$($(1)_DST_PATH)/$(1).log, "[ TARGET DEPENDS ] : [$?] ")
|
||
endef
|
||
|
||
|
||
|
||
# It invokes the Load Cache macro if cache is enabled globally as well as per module level
|
||
# Args:
|
||
# $(1) => target name
|
||
# $(2) => target output file name
|
||
define LOAD_CACHE
|
||
$(call SHOW_WHY,$(1))
|
||
$(if $(call CHECK_RCACHE_ENABLED,$(1)), $(call LOAD_FROM_CACHE,$(1),$(2)) )
|
||
endef
|
||
|
||
|
||
# It invokes the Save Cache if cache is enabled globally as well as per module level
|
||
# Args:
|
||
# $(1) => target name
|
||
# $(2) => target output file name
|
||
define SAVE_CACHE
|
||
$(if $(call CHECK_WCACHE_ENABLED,$(1)), $(call SAVE_INTO_CACHE,$(1),$(2)))
|
||
endef
|
||
|
||
|
||
|
||
# Set the target path for each target.
|
||
$(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(DEBS_PATH))) \
|
||
$(eval $(DEBS_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
$(foreach pkg, $(SONIC_MAKE_FILES), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FILES_PATH))) \
|
||
$(eval $(FILES_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
$(foreach pkg, $(SONIC_PYTHON_STDEB_DEBS), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(PYTHON_DEBS_PATH))) \
|
||
$(eval $(PYTHON_DEBS_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
$(foreach pkg, $(SONIC_PYTHON_WHEELS), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(PYTHON_WHEELS_PATH))) \
|
||
$(eval $(PYTHON_WHEELS_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
$(foreach pkg, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(TARGET_PATH))) \
|
||
$(eval $(TARGET_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
$(foreach pkg, $(SONIC_INSTALL_PKGS), \
|
||
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FSROOT_PATH))) \
|
||
$(eval $(FSROOT_PATH)/$(pkg)_TARGET := $(pkg)) )
|
||
|
||
|
||
# define the DEP files(.dep and .smdep) and SHA files (.sha and smsha) for each target
|
||
$(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS) \
|
||
$(SONIC_MAKE_FILES) $(SONIC_PYTHON_STDEB_DEBS) $(SONIC_PYTHON_WHEELS) \
|
||
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS), \
|
||
$(eval $(pkg)_MOD_SRC_PATH:=$(if $($(pkg)_SRC_PATH),$($(pkg)_SRC_PATH),$($(pkg)_PATH))) \
|
||
$(eval $(pkg)_BASE_PATH:=$(if $($(pkg)_BASE_PATH),$($(pkg)_BASE_PATH),$(CURDIR))) \
|
||
$(eval $(pkg)_DEP_FLAGS_FILE:=$($(pkg)_DST_PATH)/$(pkg).flags) \
|
||
$(eval $(pkg)_MOD_DEP_FILE:=$($(pkg)_DST_PATH)/$(pkg).dep) \
|
||
$(eval $(pkg)_MOD_HASH_FILE:=$($(pkg)_DST_PATH)/$(pkg).dep.sha) \
|
||
$(eval $(pkg)_SMOD_DEP_FILE:=$(if $($(pkg)_SMDEP_FILES),$($(pkg)_DST_PATH)/$(pkg).smdep)) \
|
||
$(eval $(pkg)_SMOD_HASH_FILE:=$(if $($(pkg)_SMDEP_FILES),$($(pkg)_DST_PATH)/$(pkg).smdep.smsha)) \
|
||
$(eval $(pkg)_DEP_FILES_LIST := $($(pkg)_DEP_FLAGS_FILE) $($(pkg)_DEP_FILES) $($(pkg)_SMDEP_FILES)) \
|
||
$(eval $(pkg)_CACHE_DIR := $(SONIC_DPKG_CACHE_DIR)) \
|
||
$(if $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)), \
|
||
$(if $(filter-out none,$($(pkg)_CACHE_MODE)), \
|
||
$(if $($(pkg)_SMDEP_FILES), \
|
||
$(if $($(pkg)_SMDEP_PATHS),,$(info Missing PATH/SRC_PATH attribute for $(pkg) package)) \
|
||
),\
|
||
$(info [ DPKG ] Cache is not enabled for $(pkg) package)\
|
||
)\
|
||
) \
|
||
)
|
||
|
||
|
||
# DPGK framework creates three dependency files for each target.
|
||
# 1. Flags file (.flags)
|
||
# 2. Dependency file (.dep),
|
||
# 3. Dependecy SHA hash file (.sha)
|
||
# 4. If the target is a submodule, corresponding dependency file and hash file are created
|
||
# sub module dependency file (.smdep)
|
||
# sub module hash file (.smsha)
|
||
# For example: following are the cache framework files for bash module
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb => Final debian package
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb.flags => Environment Flag file
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb.dep => Dependency files list
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb.dep.sha => SHA Hash file
|
||
#
|
||
#
|
||
# [1] .flags => contains value of all the environment variables of a target.
|
||
# Each target can have dependency with one or more environment variable.
|
||
# For example:
|
||
# SONIC_DEBUGGING_ON=y
|
||
# SONIC_PROFILING_ON=y
|
||
# SONIC_SANITIZER_ON=y
|
||
# etc
|
||
# If any of the ENV flag variables are modified, the target needs to be rebuilt as
|
||
# the content of flag file is changed becase of value of ENV variable is changed.
|
||
#
|
||
# [2] .dep => contains the dependency files list for a target. Eeach traget can have one or more dependency files.
|
||
# If any of the ENV flag variables are modified, the target needs to be rebuilt.
|
||
# For example: Dependency files list for 'bash' module
|
||
# rules/bash.mk
|
||
# rules/bash.dep
|
||
# src/bash/Makefile
|
||
# etc
|
||
#
|
||
# [3] .sha => contains the 48 byte SHA value for each dependency file present in the .dep file.
|
||
# For example:
|
||
# 9604676527653dcf7b6046fdda7ba52026b7f56f rules/bash.mk
|
||
# 191c345c1270776b3902c9ec91d5e777e0b5e2a3 rules/bash.dep
|
||
# 55692fe59303554b5958b04aa62c3651bc34bb6a src/bash/Makefile
|
||
# etc
|
||
#
|
||
# If module target is a sub module in the sonic repo, the following additional files gets created for that target.
|
||
# .smdep => contains the dependency files list
|
||
# .smsha => contains the SHA hash value for .smdep files list.
|
||
|
||
|
||
|
||
|
||
|
||
# ruiles for <.flags> file creation
|
||
#
|
||
# Each target defines a variable called '_DEP_FLAGS' that contais a list of environment flags for that target and
|
||
# that indicates that target needs to be rebuilt if any of the dependent flags are changed.
|
||
# An environmental dependency flags file is created with the name as ‘<target name>.flags’ for each target.
|
||
# This file contains the values of target environment flags and gets updated only when there is a change in the flag's value.
|
||
# This file is added as a dependency to the target, so that any change in the file will trigger the target recompilation.
|
||
# For Eg:
|
||
# target/debs/stretch/linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb.flags
|
||
#
|
||
# RULE args:
|
||
# $(1) => target name
|
||
# $(2) => target destination folder path
|
||
# $(3) => target file extension
|
||
# $(4) => additional flags
|
||
#
|
||
# It updates the _DEP_FLAGS variable if there is any change in the module flags.
|
||
|
||
define FLAGS_DEP_RULES
|
||
ALL_DEP_FILES_LIST += $(foreach pkg,$(2), $(if $(filter none,$($(1)_CACHE_MODE)),$(addsuffix .$(3),$(addprefix $(pkg)/, $(1)))))
|
||
$(addsuffix .$(3),$(addprefix $(2)/, $(1))) :: $(2)/%.$(3) :
|
||
@$$(eval $$*_FILE_FLAGS := $$(shell test -f $$@ && cat $$@))
|
||
@$$(eval $$*_DEP_FLAGS_ALL := $$(shell echo '$$($$*_DEP_FLAGS) $(4)' | sed -E 's/[ ]+/ /g' | sed -E 's/[ ]+$$$$//g'))
|
||
@echo '$$($$*_DEP_FLAGS_ALL)' | cmp -s - $$@ || echo '$$($$*_DEP_FLAGS_ALL)' > $$@
|
||
$$(eval $$*_FLAGS_DIFF := $$(filter-out $$($$*_FILE_FLAGS),$$($$*_DEP_FLAGS_ALL)) $$(filter-out $$($$*_DEP_FLAGS_ALL),$$($$*_FILE_FLAGS)))
|
||
@$$(if $$(MDEBUG), $$(info FLAGS: $$@, DEP:$$?))
|
||
endef
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS), $(DEBS_PATH),flags,$(BLDENV)) )
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_MAKE_FILES), $(FILES_PATH),flags,$(BLDENV)))
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),flags))
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),flags))
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),flags))
|
||
$(eval $(call FLAGS_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),flags))
|
||
|
||
|
||
|
||
|
||
# rules for <.smdep> and <.smsha> file creation
|
||
# This rule creates two dependency files for a target if the target is a submodule
|
||
# [1] .smdep file
|
||
# Each module target defines a variable called '_SMDEP_FILES' that contains the list of sub module dependency files for the target.
|
||
# Contents of the '_SMDEP_FILES' variable is stored in this file
|
||
#
|
||
# [2] .smsha file
|
||
# The SHA hash (.smsha) file is created from 48 byte SHA value for each of the dependency files present in the .smdep file
|
||
#
|
||
# The target needs to be rebuilt if any of the dependent flags are changed.
|
||
# The submodule dependency file is created with the name as '<target name>.smdep' and the SHA hash file created as '<target name>.smdep.smsha'.
|
||
# This file is added as a dependency to the target, so that any change in the file will trigger the target recompilation.
|
||
# For Eg:
|
||
# target/debs/stretch/linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb.smdep
|
||
# target/debs/stretch/linux-headers-4.9.0-9-2-common_4.9.168-1+deb9u3_all.deb.smdep.smsha
|
||
#
|
||
# RULE args:
|
||
# $(1) => target name
|
||
# $(2) => target destination folder path
|
||
# $(3) => target file extension
|
||
|
||
define SMSHA_DEP_RULES
|
||
ALL_DEP_FILES_LIST += $(foreach pkg,$(2), $($(filter none,$($(1)_CACHE_MODE)), \
|
||
$(addsuffix .$(3),$(addprefix $(pkg)/, $(1))) \
|
||
$(addsuffix .$(3).smsha,$(addprefix $(pkg)/, $(1)))))
|
||
$(addsuffix .$(3),$(addprefix $(2)/, $(1))) : $(2)/%.$(3) : \
|
||
$(2)/%.flags $$$$($$$$*_SMDEP_FILES)
|
||
@$$(eval $$*_SMDEP_FILES_MODIFIED := $$? )
|
||
@$$(file >$$@,$$(patsubst $$($$*_MOD_SRC_PATH)/%,%,$$($$*_SMDEP_FILES)))
|
||
@( cd $$($$*_MOD_SRC_PATH) ; cat $$($$*_BASE_PATH)/$$@ |xargs git hash-object ) >$$@.smsha
|
||
@$$(if $$(MDEBUG), $$(info SMDEP:$$@, MOD:$$?))
|
||
endef
|
||
$(eval $(call SMSHA_DEP_RULES, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS), $(DEBS_PATH),smdep))
|
||
$(eval $(call SMSHA_DEP_RULES, $(SONIC_MAKE_FILES), $(FILES_PATH),smdep))
|
||
$(eval $(call SMSHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),smdep))
|
||
$(eval $(call SMSHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),smdep))
|
||
$(eval $(call SMSHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),smdep))
|
||
|
||
|
||
|
||
|
||
|
||
# rules for <.dep> and <.sha> file creation
|
||
#
|
||
# This rule creates two dependency files for the target
|
||
# [1] .dep file
|
||
# Each module target defines a variable called '_DEP_FILES' that contains the list of dependency files for the target.
|
||
# Contents of the '_DEP_FILES' variable is stored in this file
|
||
#
|
||
# [2] .sha file
|
||
# The SHA hash (.sha) file is created from 48 byte SHA value for each of the dependency files present in the .dep file
|
||
#
|
||
# The target needs to be rebuilt if any of the dependent flags are changed.
|
||
# The module dependency file is created with the name as '<target name>.dep' and the SHA hash file created as '<target name>.dep.sha'.
|
||
# This file is added as a dependency to the target, so that any change in the file will trigger the target recompilation.
|
||
# For Eg:
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb.dep
|
||
# target/debs/stretch/bash_4.3-14_amd64.deb.dep.sha
|
||
#
|
||
# RULE args:
|
||
# $(1) => target name
|
||
# $(2) => target destination folder path
|
||
# $(3) => target file extension
|
||
#
|
||
|
||
define SHA_DEP_RULES
|
||
ALL_DEP_FILES_LIST += $(foreach pkg,$(2), $($(filter none,$($(1)_CACHE_MODE)), \
|
||
$(addsuffix .$(3),$(addprefix $(pkg)/, $(1))) \
|
||
$(addsuffix .$(3).sha,$(addprefix $(pkg)/, $(1)))))
|
||
$(foreach docker, $(filter $(SONIC_DOCKER_IMAGES), $(1)), \
|
||
$(eval $(docker)_DEP_FILES+=$(wildcard files/build/versions/default/*) \
|
||
$(wildcard files/build/versions/dockers/$(basename $(docker))/*)))
|
||
$(foreach docker, $(filter $(SONIC_DOCKER_DBG_IMAGES), $(1)), \
|
||
$(eval $(docker)_DEP_FILES+=$(wildcard files/build/versions/default/*) \
|
||
$(wildcard files/build/versions/dockers/$(patsubst %-$(DBG_IMAGE_MARK).gz,%,$(docker))/*)))
|
||
$(addsuffix .$(3),$(addprefix $(2)/, $(1))) : $(2)/%.$(3) : \
|
||
$(2)/%.flags $$$$($$$$*_DEP_FILES) $$$$(if $$$$($$$$*_SMDEP_FILES), $(2)/%.smdep)
|
||
@$$(eval $$*_DEP_FILES_MODIFIED := $$? )
|
||
@$$(file >$$@.tmp,$$($$*_DEP_FILES))
|
||
@cat $$@.tmp |xargs git hash-object >$$@.sha.tmp
|
||
@if ! cmp -s $$@.sha.tmp $$@.sha; then cp $$@.tmp $$@; cp $$@.sha.tmp $$@.sha; fi
|
||
@rm -f $$@.tmp $$@.sha.tmp
|
||
@$$(if $$(MDEBUG), $$(info DEP: $$@, MOD:$$?))
|
||
endef
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS), $(DEBS_PATH),dep))
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_MAKE_FILES), $(FILES_PATH),dep))
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),dep))
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),dep))
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),dep))
|
||
$(eval $(call SHA_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),dep))
|
||
|
||
|
||
|
||
|
||
|
||
# Clean all the DEP and SHA files for all the DEBS target
|
||
SONIC_CACHE_CLEAN_DEBS = $(addsuffix -clean,$(addprefix $(DEBS_PATH)/, \
|
||
$(SONIC_ONLINE_DEBS) \
|
||
$(SONIC_COPY_DEBS) \
|
||
$(SONIC_MAKE_DEBS) \
|
||
$(SONIC_DPKG_DEBS) \
|
||
$(SONIC_DERIVED_DEBS) \
|
||
$(SONIC_EXTRA_DEBS)))
|
||
$(SONIC_CACHE_CLEAN_DEBS) :: $(DEBS_PATH)/%-clean : .platform $$(addsuffix -clean,$$(addprefix $(DEBS_PATH)/,$$($$*_MAIN_DEB)))
|
||
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
|
||
$($*_MOD_DEP_FILE) $($*_SMOD_DEP_FILE)
|
||
|
||
|
||
# Clean all the DEP and SHA files for all the FILES target
|
||
SONIC_CACHE_CLEAN_FILES = $(addsuffix -clean,$(addprefix $(FILES_PATH)/, \
|
||
$(SONIC_ONLINE_FILES) \
|
||
$(SONIC_COPY_FILES) \
|
||
$(SONIC_MAKE_FILES)))
|
||
$(SONIC_CACHE_CLEAN_FILES) :: $(FILES_PATH)/%-clean : .platform
|
||
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
|
||
$($*_MOD_DEP_FILE) $($*_SMOD_DEP_FILE)
|
||
|
||
|
||
# Clean all the DEP and SHA files for all the DOCKER target
|
||
SONIC_CACHE_CLEAN_TARGETS = $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
|
||
$(SONIC_DOCKER_IMAGES) \
|
||
$(SONIC_DOCKER_DBG_IMAGES) \
|
||
$(SONIC_SIMPLE_DOCKER_IMAGES) \
|
||
$(SONIC_INSTALLERS)))
|
||
$(SONIC_CACHE_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
|
||
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
|
||
$($*_MOD_DEP_FILE) $($*_SMOD_DEP_FILE)
|
||
|
||
|
||
# Clean all the DEP and SHA files for all the PYTHON DEBS target
|
||
SONIC_CACHE_CLEAN_STDEB_DEBS = $(addsuffix -clean,$(addprefix $(PYTHON_DEBS_PATH)/, \
|
||
$(SONIC_PYTHON_STDEB_DEBS)))
|
||
$(SONIC_CACHE_CLEAN_STDEB_DEBS) :: $(PYTHON_DEBS_PATH)/%-clean : .platform
|
||
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
|
||
$($*_MOD_DEP_FILE) $($*_SMOD_DEP_FILE)
|
||
|
||
|
||
# Clean all the DEP and SHA files for all the PYTHON WHEELS target
|
||
SONIC_CACHE_CLEAN_WHEELS = $(addsuffix -clean,$(addprefix $(PYTHON_WHEELS_PATH)/, \
|
||
$(SONIC_PYTHON_WHEELS)))
|
||
$(SONIC_CACHE_CLEAN_WHEELS) :: $(PYTHON_WHEELS_PATH)/%-clean : .platform
|
||
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
|
||
$($*_MOD_DEP_FILE) $($*_SMOD_DEP_FILE)
|
||
|
||
.PHONY: cclean
|
||
cclean:: $(SONIC_CACHE_CLEAN_DEBS) $(SONIC_CACHE_CLEAN_FILES) $(SONIC_CACHE_CLEAN_TARGETS) \
|
||
$(SONIC_CACHE_CLEAN_STDEB_DEBS) $(SONIC_CACHE_CLEAN_WHEELS)
|
||
|
||
.PHONY: clean
|
||
clean:: cclean
|
||
|
||
# Clear all the local cache contents
|
||
.PHONY:lcclean
|
||
lcclean::
|
||
@rm -f $(TARGET_PATH)/cache/*
|
||
|
||
|
||
|
||
# List all main targets and its derived target with indent.
|
||
listall :
|
||
@$(foreach target,$(SONIC_TARGET_LIST),\
|
||
$(eval DPKG:=$(lastword $(subst /, ,$(target)))) \
|
||
$(eval PATH:= $(subst $(DPKG),,$(target))) \
|
||
$(if $($(DPKG)_MAIN_DEB),,
|
||
echo "[$(target)] "; \
|
||
$(foreach pkg,$($(DPKG)_DERIVED_DEBS) $($(DPKG)_EXTRA_DEBS),\
|
||
echo " $(PATH)$(pkg)"; \
|
||
)\
|
||
)\
|
||
)
|
||
|
||
#$(addprefix show-,$(SONIC_TARGET_LIST)):show-%:
|
||
show-%:
|
||
@$(foreach target,$(SONIC_TARGET_LIST),\
|
||
$(eval DPKG:=$(lastword $(subst /, ,$(target)))) \
|
||
$(eval PATH:= $(subst $(DPKG),,$(target))) \
|
||
$(if $(findstring $*,$(target)),
|
||
$(info ) \
|
||
$(eval MDPKG:=$(if $($(DPKG)_MAIN_DEB),$($(DPKG)_MAIN_DEB),$(DPKG))) \
|
||
$(info [$(PATH)$(MDPKG)] ) \
|
||
$(foreach pkg,$($(MDPKG)_DERIVED_DEBS) $($(MDPKG)_EXTRA_DEBS),\
|
||
$(info $(SPACE)$(SPACE)$(SPACE)$(SPACE) $(PATH)$(pkg)) \
|
||
)\
|
||
)\
|
||
)
|
||
$(info )
|
||
|
||
|
||
|
||
# Cache prune - Remove least frequently used cache files.
|
||
NUMDAYS ?= 7 # Delete all the cache files which are not used within last 7 days
|
||
.PHONY: cprune
|
||
cprune:
|
||
@find $(SONIC_DPKG_CACHE_DIR) -name "*.tgz" ! -mtime -$(NUMDAYS) -exec rm -f {} \;
|
||
|
||
|
||
|
||
# Invoke DPKG dependency only if DPKG cache is enabled.
|
||
define dpkg_depend
|
||
$(if $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$(1))
|
||
endef
|