sonic-buildimage/scripts/populate_file_web_server.py

#!/usr/bin/env python3

import sys
import os
import time
import argparse
from http import HTTPStatus
try:
    import requests
except ImportError:
    print("requests module is not installed. script will fail to execute")

#debug print level
PRINT_LEVEL_ERROR         = "err"
PRINT_LEVEL_WARN          = "warn"
PRINT_LEVEL_INFO          = "info"
PRINT_LEVEL_VERBOSE       = "verbose"

PRINT_LEVEL_LUT = {PRINT_LEVEL_ERROR    : 1,
                   PRINT_LEVEL_WARN     : 2,
                   PRINT_LEVEL_INFO     : 3,
                   PRINT_LEVEL_VERBOSE  : 4 }

#return code
RET_CODE_SUCCESS                = 0
RET_CODE_CANNOT_CREATE_FILE     = -1
RET_CODE_CANNOT_OPEN_FILE       = -2
RET_CODE_HTTP_SERVER_ERROR      = -3
RET_CODE_CANNOT_WRITE_FILE      = -4

#constants
RESOURCES_FILE_NAME       = 'versions-web'
EXCLUDE_DIRECTORES        = ['fsroot', 'target']
HASH_SEPARATOR            = '-'
DEFAULT_INVALID_INPUT     = 'none'

# global variables
g_current_print_level     = PRINT_LEVEL_INFO

#Script debug features (disabled by default)
g_delete_resources_in_cache = True


# global Classes
class Resource:
    def __init__(self, line, file):
        self.file = file
        temp=line.split("==")
        assert(2==len(temp))
        self.url=temp[0].strip()
        self.hash=temp[1].strip()
        temp=self.url.split("/")
        assert(len(temp)>0)
        self.name=temp[len(temp)-1]
        #handle special scenarios
        if 0 != self.name.count('?') == True:
            temp = self.name.split("?")
            self.name = temp[0]

    def get_unique_name(self):
        return self.name + HASH_SEPARATOR + self.hash

    def get_url(self):
        return self.url

    def __str__(self):
        ret_val = "Resource name: " + self.name + "\n"
        ret_val += "File: " + self.file + "\n"
        ret_val += "Hash: " + self.hash + "\n"
        ret_val += "Full URL: " + self.url
        return ret_val

# Helper functions

def print_msg(print_level, msg, print_in_place=False):
    if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[print_level]:
        if True == print_in_place:
            print(msg, end='\r')
        else:
            print(msg)

def create_dir_if_not_exist(dir):
    if not os.path.exists(dir):
        try:
            os.makedirs(dir)
        except:
            print_msg(PRINT_LEVEL_WARN, "Cannot create directory " + dir)

def delete_file_if_exist(file):
    if os.path.exists(file):
        try:
            os.remove(file)
        except:
            print_msg(PRINT_LEVEL_WARN, "Cannot delete " + file)

# Logic functions

def generate_output_file(resources, dest_url_valid, dest_url, output_file_name):
    try:
        with open(output_file_name, 'w') as f:
            for unique_name in resources.keys():
                resource = resources[unique_name]
                if True == dest_url_valid:
                    line = dest_url
                else:
                    line = resource.get_url()
                if line[-1] != '/':
                    line += '/'
                line += resource.name + "==" + resource.hash
                f.write(line + '\n')
    except:
        print_msg(PRINT_LEVEL_WARN, output_file_name + " cannot be created")
        return RET_CODE_CANNOT_CREATE_FILE

    return RET_CODE_SUCCESS

def upload_resource_to_server(resource_path, resource_name, user, key, server_url):
    url_full_path = server_url + "/" + resource_name

    try:
        f = open(resource_path, 'rb')
    except:
        err_print("Cannot open " + resource_path)
        return RET_CODE_CANNOT_OPEN_FILE

    headers = {'Content-type': 'application', 'Slug': resource_name}
    response = requests.put(url_full_path, data=f,
                            headers=headers, auth=(user, key))

    f.close()

    if response.status_code != HTTPStatus.CREATED.value:
        err_print(f"HTTP request returned status code {response.status_code}, expected {HTTPStatus.CREATED.value}")
        return RET_CODE_HTTP_SERVER_ERROR

    # JSON response empty only when status code is 204
    reported_md5 = response.json().get('checksums', {}).get('md5')
    file_md5 = resource_name.split(HASH_SEPARATOR)[-1]

    # Check if server reports checksum, if so compare reported sum and the one
    # specified in filename
    if reported_md5 != None and reported_md5 != file_md5:
        print_msg(PRINT_LEVEL_WARN, f"Server reported file's chsum {reported_md5}, expected {file_md5}")


    return RET_CODE_SUCCESS

def download_external_resouce(resource, cache_path):
    resource_path_in_cache = cache_path + os.sep + resource.get_unique_name()

    r = requests.get(resource.get_url(), allow_redirects=True)

    try:
        f = open(resource_path_in_cache, 'wb')
        f.write(r.content)
        f.close()
    except:
        print_msg(PRINT_LEVEL_ERROR, "Cannot write " + resource_path_in_cache + " to cache")
        resource_path_in_cache = "" #report error

    return resource_path_in_cache

def get_resources_list(resource_files_list):
    resource_list = list()

    for file_name in resource_files_list:
        try:
            with open(file_name, 'r') as f:
                for line in f:
                    resource_list.append(Resource(line, file_name))
        except:
            print_msg(PRINT_LEVEL_WARN, file_name + " cannot be opened")

    return resource_list

def filter_out_dir(subdir):
    ret_val = False

    for exclude in EXCLUDE_DIRECTORES:
        if exclude in subdir.split(os.sep):
            ret_val = True
            break

    return ret_val

def get_resource_files_list(serach_path):
    resource_files_list = list()

    for subdir, dirs, files in os.walk(serach_path):
        for file in files:
            if False == filter_out_dir(subdir) and RESOURCES_FILE_NAME == file:
                file_full_path = os.path.join(subdir, file)
                print_msg(PRINT_LEVEL_VERBOSE, "Found resource file :" + file_full_path)
                resource_files_list.append(file_full_path)

    return resource_files_list

def parse_args():
    parser = argparse.ArgumentParser(description='Various pre-steps for build compilation')

    parser.add_argument('-s', '--source', default=".",
                        help='Search path for ' + RESOURCES_FILE_NAME + ' files')

    parser.add_argument('-c', '--cache', default="." + os.sep + "tmp",
                        help='Path to cache for storing content before uploading to server')

    parser.add_argument('-p', '--print', default=PRINT_LEVEL_INFO,
                        choices=[PRINT_LEVEL_ERROR, PRINT_LEVEL_WARN, PRINT_LEVEL_INFO, PRINT_LEVEL_VERBOSE],
                        help='Print level verbosity')

    parser.add_argument('-o', '--output', default=DEFAULT_INVALID_INPUT,
                        help='Output file name to hold the list of packages')

    parser.add_argument('-u', '--user', default=DEFAULT_INVALID_INPUT,
                        help='User for server authentication')

    parser.add_argument('-k', '--key', default=DEFAULT_INVALID_INPUT,
                        help='API key server authentication')

    parser.add_argument('-d', '--dest', default=DEFAULT_INVALID_INPUT,
                        help='URL for destination web file server')

    return parser.parse_args()

def main():
    global g_current_print_level
    ret_val = RET_CODE_SUCCESS
    resource_counter = 0.0
    resource_dict = dict()

    args = parse_args()

    g_current_print_level = args.print

    resource_files_list = get_resource_files_list(args.source)

    resource_list = get_resources_list(resource_files_list)

    #remove duplications
    for resource in resource_list:
        unique_name = resource.get_unique_name()
        if not unique_name in resource_dict.keys():
            resource_dict[unique_name] = resource

    print_msg(PRINT_LEVEL_INFO, "Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources")

    if args.dest != DEFAULT_INVALID_INPUT:
        upload_files_to_server = True
        print_msg(PRINT_LEVEL_INFO, "Upload files to URL - " + args.dest)
    else:
        upload_files_to_server = False
        print_msg(PRINT_LEVEL_INFO, "Skipping files upload to server")

    #create cache directory if not exist
    create_dir_if_not_exist(args.cache)

    #download content to cache and then upload to web server
    for unique_name in resource_dict.keys():

        resource = resource_dict[unique_name]

        print_msg(PRINT_LEVEL_VERBOSE, resource)

        resource_counter += 1.0

        #download content to cache
        file_in_cache = download_external_resouce(resource, args.cache)

        if "" == file_in_cache:
            return RET_CODE_CANNOT_WRITE_FILE

        if True == upload_files_to_server:
            #upload content to web server
            ret_val = upload_resource_to_server(file_in_cache, unique_name, args.user, args.key, args.dest)
            if ret_val != RET_CODE_SUCCESS:
                return ret_val

        if True == g_delete_resources_in_cache:
            delete_file_if_exist(file_in_cache)

        print_msg(PRINT_LEVEL_INFO, "Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%", True) #print progress bar

    # generate version output file as needed
    if args.output != DEFAULT_INVALID_INPUT:
        ret_val = generate_output_file(resource_dict, upload_files_to_server, args.dest, args.output)
        print_msg(PRINT_LEVEL_INFO, "Generate output file " + args.output)

    return ret_val

#    Entry function
if __name__ == '__main__':

    ret_val = main()

    sys.exit(ret_val)