#!/usr/bin/env python3 import sys import os import time import argparse from http import HTTPStatus try: import requests except ImportError: print("requests module is not installed. script will fail to execute") #debug print level PRINT_LEVEL_ERROR = "err" PRINT_LEVEL_WARN = "warn" PRINT_LEVEL_INFO = "info" PRINT_LEVEL_VERBOSE = "verbose" PRINT_LEVEL_LUT = {PRINT_LEVEL_ERROR : 1, PRINT_LEVEL_WARN : 2, PRINT_LEVEL_INFO : 3, PRINT_LEVEL_VERBOSE : 4 } #return code RET_CODE_SUCCESS = 0 RET_CODE_CANNOT_CREATE_FILE = -1 RET_CODE_CANNOT_OPEN_FILE = -2 RET_CODE_HTTP_SERVER_ERROR = -3 RET_CODE_CANNOT_WRITE_FILE = -4 #constants RESOURCES_FILE_NAME = 'versions-web' EXCLUDE_DIRECTORES = ['fsroot', 'target'] HASH_SEPARATOR = '-' DEFAULT_INVALID_INPUT = 'none' # global variables g_current_print_level = PRINT_LEVEL_INFO #Script debug features (disabled by default) g_delete_resources_in_cache = True # global Classes class Resource: def __init__(self, line, file): self.file = file temp=line.split("==") assert(2==len(temp)) self.url=temp[0].strip() self.hash=temp[1].strip() temp=self.url.split("/") assert(len(temp)>0) self.name=temp[len(temp)-1] #handle special scenarios if 0 != self.name.count('?') == True: temp = self.name.split("?") self.name = temp[0] def get_unique_name(self): return self.name + HASH_SEPARATOR + self.hash def get_url(self): return self.url def __str__(self): ret_val = "Resource name: " + self.name + "\n" ret_val += "File: " + self.file + "\n" ret_val += "Hash: " + self.hash + "\n" ret_val += "Full URL: " + self.url return ret_val # Helper functions def print_msg(print_level, msg, print_in_place=False): if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[print_level]: if True == print_in_place: print(msg, end='\r') else: print(msg) def create_dir_if_not_exist(dir): if not os.path.exists(dir): try: os.makedirs(dir) except: print_msg(PRINT_LEVEL_WARN, "Cannot create directory " + dir) def delete_file_if_exist(file): if os.path.exists(file): try: os.remove(file) except: print_msg(PRINT_LEVEL_WARN, "Cannot delete " + file) # Logic functions def generate_output_file(resources, dest_url_valid, dest_url, output_file_name): try: with open(output_file_name, 'w') as f: for unique_name in resources.keys(): resource = resources[unique_name] if True == dest_url_valid: line = dest_url else: line = resource.get_url() if line[-1] != '/': line += '/' line += resource.name + "==" + resource.hash f.write(line + '\n') except: print_msg(PRINT_LEVEL_WARN, output_file_name + " cannot be created") return RET_CODE_CANNOT_CREATE_FILE return RET_CODE_SUCCESS def upload_resource_to_server(resource_path, resource_name, user, key, server_url): url_full_path = server_url + "/" + resource_name try: f = open(resource_path, 'rb') except: err_print("Cannot open " + resource_path) return RET_CODE_CANNOT_OPEN_FILE headers = {'Content-type': 'application', 'Slug': resource_name} response = requests.put(url_full_path, data=f, headers=headers, auth=(user, key)) f.close() if response.status_code != HTTPStatus.CREATED.value: err_print(f"HTTP request returned status code {response.status_code}, expected {HTTPStatus.CREATED.value}") return RET_CODE_HTTP_SERVER_ERROR # JSON response empty only when status code is 204 reported_md5 = response.json().get('checksums', {}).get('md5') file_md5 = resource_name.split(HASH_SEPARATOR)[-1] # Check if server reports checksum, if so compare reported sum and the one # specified in filename if reported_md5 != None and reported_md5 != file_md5: print_msg(PRINT_LEVEL_WARN, f"Server reported file's chsum {reported_md5}, expected {file_md5}") return RET_CODE_SUCCESS def download_external_resouce(resource, cache_path): resource_path_in_cache = cache_path + os.sep + resource.get_unique_name() r = requests.get(resource.get_url(), allow_redirects=True) try: f = open(resource_path_in_cache, 'wb') f.write(r.content) f.close() except: print_msg(PRINT_LEVEL_ERROR, "Cannot write " + resource_path_in_cache + " to cache") resource_path_in_cache = "" #report error return resource_path_in_cache def get_resources_list(resource_files_list): resource_list = list() for file_name in resource_files_list: try: with open(file_name, 'r') as f: for line in f: resource_list.append(Resource(line, file_name)) except: print_msg(PRINT_LEVEL_WARN, file_name + " cannot be opened") return resource_list def filter_out_dir(subdir): ret_val = False for exclude in EXCLUDE_DIRECTORES: if exclude in subdir.split(os.sep): ret_val = True break return ret_val def get_resource_files_list(serach_path): resource_files_list = list() for subdir, dirs, files in os.walk(serach_path): for file in files: if False == filter_out_dir(subdir) and RESOURCES_FILE_NAME == file: file_full_path = os.path.join(subdir, file) print_msg(PRINT_LEVEL_VERBOSE, "Found resource file :" + file_full_path) resource_files_list.append(file_full_path) return resource_files_list def parse_args(): parser = argparse.ArgumentParser(description='Various pre-steps for build compilation') parser.add_argument('-s', '--source', default=".", help='Search path for ' + RESOURCES_FILE_NAME + ' files') parser.add_argument('-c', '--cache', default="." + os.sep + "tmp", help='Path to cache for storing content before uploading to server') parser.add_argument('-p', '--print', default=PRINT_LEVEL_INFO, choices=[PRINT_LEVEL_ERROR, PRINT_LEVEL_WARN, PRINT_LEVEL_INFO, PRINT_LEVEL_VERBOSE], help='Print level verbosity') parser.add_argument('-o', '--output', default=DEFAULT_INVALID_INPUT, help='Output file name to hold the list of packages') parser.add_argument('-u', '--user', default=DEFAULT_INVALID_INPUT, help='User for server authentication') parser.add_argument('-k', '--key', default=DEFAULT_INVALID_INPUT, help='API key server authentication') parser.add_argument('-d', '--dest', default=DEFAULT_INVALID_INPUT, help='URL for destination web file server') return parser.parse_args() def main(): global g_current_print_level ret_val = RET_CODE_SUCCESS resource_counter = 0.0 resource_dict = dict() args = parse_args() g_current_print_level = args.print resource_files_list = get_resource_files_list(args.source) resource_list = get_resources_list(resource_files_list) #remove duplications for resource in resource_list: unique_name = resource.get_unique_name() if not unique_name in resource_dict.keys(): resource_dict[unique_name] = resource print_msg(PRINT_LEVEL_INFO, "Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources") if args.dest != DEFAULT_INVALID_INPUT: upload_files_to_server = True print_msg(PRINT_LEVEL_INFO, "Upload files to URL - " + args.dest) else: upload_files_to_server = False print_msg(PRINT_LEVEL_INFO, "Skipping files upload to server") #create cache directory if not exist create_dir_if_not_exist(args.cache) #download content to cache and then upload to web server for unique_name in resource_dict.keys(): resource = resource_dict[unique_name] print_msg(PRINT_LEVEL_VERBOSE, resource) resource_counter += 1.0 #download content to cache file_in_cache = download_external_resouce(resource, args.cache) if "" == file_in_cache: return RET_CODE_CANNOT_WRITE_FILE if True == upload_files_to_server: #upload content to web server ret_val = upload_resource_to_server(file_in_cache, unique_name, args.user, args.key, args.dest) if ret_val != RET_CODE_SUCCESS: return ret_val if True == g_delete_resources_in_cache: delete_file_if_exist(file_in_cache) print_msg(PRINT_LEVEL_INFO, "Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%", True) #print progress bar # generate version output file as needed if args.output != DEFAULT_INVALID_INPUT: ret_val = generate_output_file(resource_dict, upload_files_to_server, args.dest, args.output) print_msg(PRINT_LEVEL_INFO, "Generate output file " + args.output) return ret_val # Entry function if __name__ == '__main__': ret_val = main() sys.exit(ret_val)