06.Version Cache - Support for wget (#14612)
When a package is referenced from the web through wget command, it downloads the package for every build. This feature caches all the packages that are being downloaded from the web, so that subsequent build always loads the cache instead of from web.
This commit is contained in:
Kalimuthu-Velappan
GitHub
parent
b26445cf7b
commit
9dce453552
@ -152,57 +152,127 @@ download_packages()
|
||||
{
|
||||
local parameters=("$@")
|
||||
declare -A filenames
|
||||
declare -A SRC_FILENAMES
|
||||
local url=
|
||||
local real_version=
|
||||
local SRC_FILENAME=
|
||||
local DST_FILENAME=
|
||||
|
||||
for (( i=0; i<${#parameters[@]}; i++ ))
|
||||
do
|
||||
local para=${parameters[$i]}
|
||||
local nexti=$((i+1))
|
||||
if [[ "$para" == *://* ]]; then
|
||||
if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then
|
||||
DST_FILENAME="${parameters[$nexti]}"
|
||||
elif [[ "$para" == *://* ]]; then
|
||||
local url=$para
|
||||
local real_version=
|
||||
|
||||
# Skip to use the proxy, if the url has already used the proxy server
|
||||
if [[ $url == ${URL_PREFIX}* ]]; then
|
||||
if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then
|
||||
continue
|
||||
fi
|
||||
local result=0
|
||||
WEB_CACHE_PATH=${PKG_CACHE_PATH}/web
|
||||
mkdir -p ${WEB_CACHE_PATH}
|
||||
local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
|
||||
if [ -z "${DST_FILENAME}" ];then
|
||||
DST_FILENAME="${WEB_FILENAME}"
|
||||
fi
|
||||
local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
|
||||
if [ ! -z "${VERSION}" ]; then
|
||||
|
||||
if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
|
||||
if [ ! -z "$(get_version_cache_option)" ]; then
|
||||
SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz
|
||||
if [ -f "${SRC_FILENAME}" ]; then
|
||||
log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}"
|
||||
cp "${SRC_FILENAME}" "${DST_FILENAME}"
|
||||
touch "${SRC_FILENAME}"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
|
||||
local version=
|
||||
local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
|
||||
[ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
|
||||
if [ -z "$version" ]; then
|
||||
log_err "Warning: Failed to verify the package: $url, the version is not specified"
|
||||
continue
|
||||
fi
|
||||
|
||||
local version_filename="${filename}-${version}"
|
||||
local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
|
||||
local url_exist=$(check_if_url_exist $proxy_url)
|
||||
if [ "$url_exist" == y ]; then
|
||||
parameters[$i]=$proxy_url
|
||||
filenames[$version_filename]=$filename
|
||||
real_version=$version
|
||||
log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
|
||||
real_version=$(get_url_version $url)
|
||||
else
|
||||
real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
|
||||
if [ "$real_version" != "$version" ]; then
|
||||
log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename"
|
||||
continue
|
||||
|
||||
local version_filename="${filename}-${version}"
|
||||
local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
|
||||
local url_exist=$(check_if_url_exist $proxy_url)
|
||||
if [ "$url_exist" == y ]; then
|
||||
parameters[$i]=$proxy_url
|
||||
filenames[$version_filename]=$filename
|
||||
real_version=$version
|
||||
else
|
||||
real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
|
||||
if [ "$real_version" != "$version" ]; then
|
||||
log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
|
||||
fi
|
||||
|
||||
# ignore md5sum for string ""
|
||||
# echo -n "" | md5sum == d41d8cd98f00b204e9800998ecf8427e
|
||||
[[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
|
||||
[[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || {
|
||||
|
||||
VERSION=${real_version}
|
||||
local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz"
|
||||
SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}"
|
||||
|
||||
echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
|
||||
sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null
|
||||
}
|
||||
fi
|
||||
done
|
||||
|
||||
# Skip the real command if all the files are loaded from cache
|
||||
local result=0
|
||||
if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then
|
||||
return $result
|
||||
fi
|
||||
|
||||
$REAL_COMMAND "${parameters[@]}"
|
||||
local result=$?
|
||||
result=$?
|
||||
|
||||
#Return if there is any error
|
||||
if [ ${result} -ne 0 ]; then
|
||||
exit ${result}
|
||||
fi
|
||||
|
||||
for filename in "${!filenames[@]}"
|
||||
do
|
||||
[ -f "$filename" ] && mv "$filename" "${filenames[$filename]}"
|
||||
if [ -f "$filename" ] ; then
|
||||
mv "$filename" "${filenames[$filename]}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$(get_version_cache_option)" ]]; then
|
||||
return $result
|
||||
fi
|
||||
|
||||
#Save them into cache
|
||||
for DST_FILENAME in "${!SRC_FILENAMES[@]}"
|
||||
do
|
||||
SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}"
|
||||
if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then
|
||||
continue
|
||||
fi
|
||||
FLOCK "${SRC_FILENAME}"
|
||||
cp "${DST_FILENAME}" "${SRC_FILENAME}"
|
||||
chmod -f 777 "${SRC_FILENAME}"
|
||||
FUNLOCK "${SRC_FILENAME}"
|
||||
log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}"
|
||||
done
|
||||
|
||||
return $result
|
||||
|
||||
Reference in New Issue
Block a user