diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index a83a5dc679..365a96b7ae 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -152,57 +152,127 @@ download_packages() { local parameters=("$@") declare -A filenames + declare -A SRC_FILENAMES + local url= + local real_version= + local SRC_FILENAME= + local DST_FILENAME= + for (( i=0; i<${#parameters[@]}; i++ )) do local para=${parameters[$i]} local nexti=$((i+1)) - if [[ "$para" == *://* ]]; then + if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then + DST_FILENAME="${parameters[$nexti]}" + elif [[ "$para" == *://* ]]; then local url=$para local real_version= # Skip to use the proxy, if the url has already used the proxy server - if [[ $url == ${URL_PREFIX}* ]]; then + if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then continue fi + local result=0 + WEB_CACHE_PATH=${PKG_CACHE_PATH}/web + mkdir -p ${WEB_CACHE_PATH} + local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + if [ -z "${DST_FILENAME}" ];then + DST_FILENAME="${WEB_FILENAME}" + fi + local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') + if [ ! -z "${VERSION}" ]; then + + if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then + if [ ! -z "$(get_version_cache_option)" ]; then + SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz + if [ -f "${SRC_FILENAME}" ]; then + log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}" + cp "${SRC_FILENAME}" "${DST_FILENAME}" + touch "${SRC_FILENAME}" + continue + fi + fi + fi + fi if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then local version= local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') if [ -z "$version" ]; then - log_err "Warning: Failed to verify the package: $url, the version is not specified" - continue - fi - - local version_filename="${filename}-${version}" - local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" - local url_exist=$(check_if_url_exist $proxy_url) - if [ "$url_exist" == y ]; then - parameters[$i]=$proxy_url - filenames[$version_filename]=$filename - real_version=$version + log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2 + real_version=$(get_url_version $url) else - real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } - if [ "$real_version" != "$version" ]; then - log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" - continue + + local version_filename="${filename}-${version}" + local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" + local url_exist=$(check_if_url_exist $proxy_url) + if [ "$url_exist" == y ]; then + parameters[$i]=$proxy_url + filenames[$version_filename]=$filename + real_version=$version + else + real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } + if [ "$real_version" != "$version" ]; then + log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 + fi fi fi else real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } fi + # ignore md5sum for string "" # echo -n "" | md5sum == d41d8cd98f00b204e9800998ecf8427e - [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || { + + VERSION=${real_version} + local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz" + SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}" + + echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null + } fi done + # Skip the real command if all the files are loaded from cache + local result=0 + if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then + return $result + fi + $REAL_COMMAND "${parameters[@]}" - local result=$? + result=$? + + #Return if there is any error + if [ ${result} -ne 0 ]; then + exit ${result} + fi for filename in "${!filenames[@]}" do - [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}" + if [ -f "$filename" ] ; then + mv "$filename" "${filenames[$filename]}" + fi + done + + if [[ -z "$(get_version_cache_option)" ]]; then + return $result + fi + + #Save them into cache + for DST_FILENAME in "${!SRC_FILENAMES[@]}" + do + SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}" + if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then + continue + fi + FLOCK "${SRC_FILENAME}" + cp "${DST_FILENAME}" "${SRC_FILENAME}" + chmod -f 777 "${SRC_FILENAME}" + FUNLOCK "${SRC_FILENAME}" + log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}" done return $result