06.Version Cache - Support for wget (#14612)

When a package is referenced from the web through wget command,
it downloads the package for every build.

This feature caches all the packages that are being downloaded from the
web, so that subsequent build always loads the cache instead of from
web.

src/sonic-build-hooks/scripts/buildinfo_base.sh

@@ -152,57 +152,127 @@ download_packages()
 {
     local parameters=("$@")
     declare -A filenames
+    declare -A SRC_FILENAMES
+    local url=
+    local real_version=
+    local SRC_FILENAME=
+    local DST_FILENAME=
+
     for (( i=0; i<${#parameters[@]}; i++ ))
     do
         local para=${parameters[$i]}
         local nexti=$((i+1))
-        if [[ "$para" == *://* ]]; then
+        if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then
+            DST_FILENAME="${parameters[$nexti]}"
+        elif [[ "$para" == *://* ]]; then
             local url=$para
             local real_version=
 
             # Skip to use the proxy, if the url has already used the proxy server
-            if [[ $url == ${URL_PREFIX}* ]]; then
+            if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then
                 continue
             fi
+            local result=0
+            WEB_CACHE_PATH=${PKG_CACHE_PATH}/web
+            mkdir -p ${WEB_CACHE_PATH}
+            local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
+            if [ -z "${DST_FILENAME}" ];then
+                DST_FILENAME="${WEB_FILENAME}"
+            fi
+            local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
+            if [ ! -z "${VERSION}" ]; then
+
+                if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
+                    if [ ! -z "$(get_version_cache_option)" ]; then
+                        SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz
+                        if [ -f "${SRC_FILENAME}" ]; then
+                            log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}"
+                            cp "${SRC_FILENAME}" "${DST_FILENAME}"
+                            touch "${SRC_FILENAME}"
+                            continue
+                        fi
+                    fi
+                fi
+            fi
 
             if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
                 local version=
                 local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
                 [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
                 if [ -z "$version" ]; then
-                    log_err "Warning: Failed to verify the package: $url, the version is not specified"
+                    log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
-                    continue
+                    real_version=$(get_url_version $url)
-                fi
-
-                local version_filename="${filename}-${version}"
-                local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
-                local url_exist=$(check_if_url_exist $proxy_url)
-                if [ "$url_exist" == y ]; then
-                    parameters[$i]=$proxy_url
-                    filenames[$version_filename]=$filename
-                    real_version=$version
                 else
-                    real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
+
-                    if [ "$real_version" != "$version" ]; then
+                    local version_filename="${filename}-${version}"
-                       log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename"
+                    local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
-                       continue
+                    local url_exist=$(check_if_url_exist $proxy_url)
+                    if [ "$url_exist" == y ]; then
+                        parameters[$i]=$proxy_url
+                        filenames[$version_filename]=$filename
+                        real_version=$version
+                    else
+                        real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
+                        if [ "$real_version" != "$version" ]; then
+                            log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
+                        fi
                     fi
                 fi
             else
                 real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
             fi
+
             # ignore md5sum for string ""
             # echo -n "" | md5sum    ==   d41d8cd98f00b204e9800998ecf8427e
-            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || {
+
+				VERSION=${real_version}
+				local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz"
+				SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}"
+
+				echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+				sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null
+			}
         fi
     done
 
+    # Skip the real command if all the files are loaded from cache
+    local result=0
+    if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then
+        return $result
+    fi
+
     $REAL_COMMAND "${parameters[@]}"
-    local result=$?
+    result=$?
+
+    #Return if there is any error
+    if [ ${result} -ne 0 ]; then
+        exit ${result}
+    fi
 
     for filename in "${!filenames[@]}"
     do
-        [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}"
+        if [ -f "$filename" ] ; then
+            mv "$filename" "${filenames[$filename]}"
+        fi
+    done
+
+    if [[  -z "$(get_version_cache_option)" ]]; then
+        return $result
+    fi
+
+    #Save them into cache
+    for DST_FILENAME in "${!SRC_FILENAMES[@]}"
+    do
+        SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}"
+        if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then
+            continue
+        fi
+        FLOCK "${SRC_FILENAME}"
+        cp "${DST_FILENAME}" "${SRC_FILENAME}"
+        chmod -f 777 "${SRC_FILENAME}"
+        FUNLOCK "${SRC_FILENAME}"
+        log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}"
     done
 
     return $result