06.Version Cache - Support for wget (#14612)

When a package is referenced from the web through wget command,
it downloads the package for every build.

This feature caches all the packages that are being downloaded from the
web, so that subsequent build always loads the cache instead of from
web.

src/sonic-build-hooks/scripts/buildinfo_base.sh

@@ -152,27 +152,57 @@ download_packages()
 {
     local parameters=("$@")
     declare -A filenames
+    declare -A SRC_FILENAMES
+    local url=
+    local real_version=
+    local SRC_FILENAME=
+    local DST_FILENAME=
+
     for (( i=0; i<${#parameters[@]}; i++ ))
     do
         local para=${parameters[$i]}
         local nexti=$((i+1))
-        if [[ "$para" == *://* ]]; then
+        if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then
+            DST_FILENAME="${parameters[$nexti]}"
+        elif [[ "$para" == *://* ]]; then
             local url=$para
             local real_version=
 
             # Skip to use the proxy, if the url has already used the proxy server
-            if [[ $url == ${URL_PREFIX}* ]]; then
+            if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then
                 continue
             fi
+            local result=0
+            WEB_CACHE_PATH=${PKG_CACHE_PATH}/web
+            mkdir -p ${WEB_CACHE_PATH}
+            local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
+            if [ -z "${DST_FILENAME}" ];then
+                DST_FILENAME="${WEB_FILENAME}"
+            fi
+            local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
+            if [ ! -z "${VERSION}" ]; then
+
+                if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
+                    if [ ! -z "$(get_version_cache_option)" ]; then
+                        SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz
+                        if [ -f "${SRC_FILENAME}" ]; then
+                            log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}"
+                            cp "${SRC_FILENAME}" "${DST_FILENAME}"
+                            touch "${SRC_FILENAME}"
+                            continue
+                        fi
+                    fi
+                fi
+            fi
 
             if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
                 local version=
                 local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
                 [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
                 if [ -z "$version" ]; then
-                    log_err "Warning: Failed to verify the package: $url, the version is not specified"
+                    log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
-                    continue
+                    real_version=$(get_url_version $url)
-                fi
+                else
 
                     local version_filename="${filename}-${version}"
                     local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
@@ -184,25 +214,65 @@ download_packages()
                     else
                         real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
                         if [ "$real_version" != "$version" ]; then
-                       log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename"
+                            log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
-                       continue
+                        fi
                     fi
                 fi
             else
                 real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
             fi
+
             # ignore md5sum for string ""
             # echo -n "" | md5sum    ==   d41d8cd98f00b204e9800998ecf8427e
-            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || {
+
+				VERSION=${real_version}
+				local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz"
+				SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}"
+
+				echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+				sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null
+			}
         fi
     done
 
+    # Skip the real command if all the files are loaded from cache
+    local result=0
+    if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then
+        return $result
+    fi
+
     $REAL_COMMAND "${parameters[@]}"
-    local result=$?
+    result=$?
+
+    #Return if there is any error
+    if [ ${result} -ne 0 ]; then
+        exit ${result}
+    fi
 
     for filename in "${!filenames[@]}"
     do
-        [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}"
+        if [ -f "$filename" ] ; then
+            mv "$filename" "${filenames[$filename]}"
+        fi
+    done
+
+    if [[  -z "$(get_version_cache_option)" ]]; then
+        return $result
+    fi
+
+    #Save them into cache
+    for DST_FILENAME in "${!SRC_FILENAMES[@]}"
+    do
+        SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}"
+        if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then
+            continue
+        fi
+        FLOCK "${SRC_FILENAME}"
+        cp "${DST_FILENAME}" "${SRC_FILENAME}"
+        chmod -f 777 "${SRC_FILENAME}"
+        FUNLOCK "${SRC_FILENAME}"
+        log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}"
     done
 
     return $result