#!/usr/bin/python3 import argparse import glob import os import sys import re ALL_DIST = 'all' ALL_ARCH = 'all' DEFAULT_MODULE = 'default' DEFAULT_VERSION_PATH = 'files/build/versions' VERSION_PREFIX="versions-" VERSION_DEB_PREFERENCE = '01-versions-deb' DEFAULT_OVERWRITE_COMPONENTS=['deb', 'py2', 'py3'] SLAVE_INDIVIDULE_VERSION = False class Component: ''' The component consists of mutiple packages ctype -- Component Type, such as deb, py2, etc dist -- Distribution, such as stretch, buster, etc arch -- Architectrue, such as amd64, arm64, etc ''' def __init__(self, verbose=None, versions={}, ctype="deb", dist=ALL_DIST, arch=ALL_ARCH): self.versions = versions self.ctype = ctype if not dist: dist = ALL_DIST if not arch: arch = ALL_ARCH self.dist = dist self.arch = arch self.verbose = verbose @classmethod def get_versions(cls, version_file): result = {} if not os.path.exists(version_file): return result with open(version_file) as fp: for line in fp.readlines(): offset = line.rfind('==') if offset > 0: package = line[:offset].strip() if 'py2' in version_file.lower() or 'py3' in version_file.lower(): package = package.lower() version = line[offset+2:].strip() result[package] = version return result def clone(self): return Component(self.verbose, self.versions.copy(), self.ctype, self.dist, self.arch) def merge(self, versions, overwritten=True): for package in versions: if overwritten or package not in self.versions: self.versions[package] = versions[package] def subtract(self, versions): for package in versions: if package in self.versions and self.versions[package] == versions[package]: del self.versions[package] def dump(self, config=False, priority=999): result = [] for package in sorted(self.versions.keys(), key=str.casefold): if config and self.ctype == 'deb': lines = 'Package: {0}\nPin: version {1}\nPin-Priority: {2}\n\n'.format(package, self.versions[package], priority) result.append(lines) else: result.append('{0}=={1}'.format(package, self.versions[package])) return "\n".join(result)+'\n' def dump_to_file(self, version_file, config=False, priority=999): if len(self.versions) <= 0: return with open(version_file, 'w') as f: f.write(self.dump(config, priority)) def dump_to_path(self, file_path, config=False, priority=999): if len(self.versions) <= 0: return if not os.path.exists(file_path): os.makedirs(file_path) filename = self.get_filename() if config and self.ctype == 'deb': none_config_file_path = os.path.join(file_path, filename) self.dump_to_file(none_config_file_path, False, priority) filename = VERSION_DEB_PREFERENCE file_path = os.path.join(file_path, filename) self.dump_to_file(file_path, config, priority) def print(self, file_path): if len(self.versions) <= 0: return if self.verbose is None: return filename = self.get_filename() file_path = os.path.join(file_path, filename) if self.verbose and re.search("cfile=", self.verbose) \ and not re.search(self.verbose, "cfile=all".format(filename)) \ and not re.search(self.verbose, "cfile={}".format(filename)): return print("VERSION : {}".format(file_path)) for package in sorted(self.versions.keys(), key=str.casefold): if self.verbose and re.search("ctype=", self.verbose) \ and not re.search("ctype=all".format(self.ctype), self.verbose) \ and not re.search("ctype={}".format(self.ctype), self.verbose): continue if self.verbose and re.search("cname=", self.verbose) \ and not re.search(self.verbose, "cname=all".format(package)) \ and not re.search(self.verbose, "cname={}".format(package)): continue if self.verbose and re.search("cver=", self.verbose) \ and not re.search(self.verbose, "cver=all".format(self.versions[package])) \ and not re.search(self.verbose, "cver={}".format(self.versions[package])): continue print('{0}=={1}'.format(package, self.versions[package])) # Check if the self component can be overwritten by the input component def check_overwritable(self, component, for_all_dist=False, for_all_arch=False): if self.ctype != component.ctype: return False if self.dist != component.dist and not (for_all_dist and self.dist == ALL_DIST): return False if self.arch != component.arch and not (for_all_arch and self.arch == ALL_ARCH): return False return True # Check if the self component can inherit the package versions from the input component def check_inheritable(self, component): if self.ctype != component.ctype: return False if self.dist != component.dist and component.dist != ALL_DIST: return False if self.arch != component.arch and component.arch != ALL_ARCH: return False return True ''' Get the file name The file name format: versions-{ctype}-{dist}-{arch} If {arch} is all, then the file name format: versions-{ctype}-{dist} if {arch} is all and {dist} is all, then the file name format: versions-{ctype} ''' def get_filename(self): filename = VERSION_PREFIX + self.ctype dist = self.dist if self.arch and self.arch != ALL_ARCH: if not dist: dist = ALL_DIST return filename + '-' + dist + '-' + self.arch if dist and self.dist != ALL_DIST: filename = filename + '-' + dist return filename def get_order_keys(self): dist = self.dist if not dist or dist == ALL_DIST: dist = '' arch = self.arch if not arch or arch == ALL_ARCH: arch = '' return (self.ctype, dist, arch) def clean_info(self, clean_dist=True, clean_arch=True, force=False): if clean_dist: if force or self.ctype != 'deb': self.dist = ALL_DIST if clean_arch: self.arch = ALL_ARCH class VersionModule: ''' The version module represents a build target, such as docker image, host image, consists of multiple components. name -- The name of the image, such as sonic-slave-buster, docker-lldp, etc ''' def __init__(self, verbose=None, name=None, components=None): self.name = name self.components = components self.module_path="" self.verbose=verbose # Overwrite the docker/host image/base image versions def overwrite(self, module, for_all_dist=False, for_all_arch=False): # Overwrite from generic one to detail one # For examples: versions-deb overwrtten by versions-deb-buster, and versions-deb-buster overwritten by versions-deb-buster-amd64 components = sorted(module.components, key = lambda x : x.get_order_keys()) for merge_component in components: merged = False for component in self.components: if component.check_overwritable(merge_component, for_all_dist=for_all_dist, for_all_arch=for_all_arch): component.merge(merge_component.versions, True) merged = True if not merged: tmp_component = merge_component.clone() tmp_component.clean_info(clean_dist=for_all_dist, clean_arch=for_all_arch) self.components.append(tmp_component) self.adjust() def get_config_module(self, source_path, dist, arch): if self.is_individule_version(): return self default_module_path = VersionModule.get_module_path_by_name(source_path, DEFAULT_MODULE) default_module = VersionModule() default_module.load(default_module_path, filter_dist=dist, filter_arch=arch) module = default_module if self.name == 'host-image': base_module_path = VersionModule.get_module_path_by_name(source_path, 'host-base-image') base_module = VersionModule() base_module.load(base_module_path, filter_dist=dist, filter_arch=arch) module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) module.overwrite(base_module, True, True) elif not self.is_aggregatable_module(self.name): module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) return self._get_config_module(module, dist, arch) #Merge the default with specific version def _get_config_module(self, default_module, dist, arch): module = default_module.clone() default_ctype_components = module._get_components_per_ctypes() module.overwrite(self) config_components = [] ctype_components = module._get_components_per_ctypes() for ctype in default_ctype_components: if ctype not in ctype_components: ctype_components[ctype] = [] for components in ctype_components.values(): if len(components) == 0: continue config_component = self._get_config_for_ctype(components, dist, arch) config_components.append(config_component) config_module = VersionModule(self.verbose, self.name, config_components) return config_module def _get_config_for_ctype(self, components, dist, arch): result = Component(self.verbose, {}, components[0].ctype, dist, arch) for component in sorted(components, key = lambda x : x.get_order_keys()): if result.check_inheritable(component): result.merge(component.versions, True) return result def subtract(self, default_module): module = self.clone() result = [] ctype_components = module._get_components_per_ctypes() for ctype in ctype_components: components = ctype_components[ctype] components = sorted(components, key = lambda x : x.get_order_keys()) for i in range(0, len(components)): component = components[i] base_module = VersionModule(self.verbose, self.name, components[0:i]) config_module = base_module._get_config_module(default_module, component.dist, component.arch) config_components = config_module._get_components_by_ctype(ctype) if len(config_components) > 0: config_component = config_components[0] component.subtract(config_component.versions) if len(component.versions): result.append(component) self.components = result def adjust(self): result_components = [] ctype_components = self._get_components_per_ctypes() for components in ctype_components.values(): result_components += self._adjust_components_for_ctype(components) self.components = result_components def _get_components_by_ctype(self, ctype): components = [] for component in self.components: if component.ctype == ctype: components.append(component) return components def _adjust_components_for_ctype(self, components): components = sorted(components, key = lambda x : x.get_order_keys()) result = [] for i in range(0, len(components)): component = components[i] inheritable_component = Component(self.verbose, {}, component.ctype) for j in range(0, i): base_component = components[j] if component.check_inheritable(base_component): inheritable_component.merge(base_component.versions, True) component.subtract(inheritable_component.versions) if len(component.versions) > 0: result.append(component) return result def _get_components_per_ctypes(self): result = {} for component in self.components: components = result.get(component.ctype, []) components.append(component) result[component.ctype] = components return result def load(self, image_path, filter_ctype=None, filter_dist=None, filter_arch=None): version_file_pattern = os.path.join(image_path, VERSION_PREFIX) + '*' file_paths = glob.glob(version_file_pattern) components = [] self.name = os.path.basename(image_path) self.module_path = image_path self.components = components for file_path in file_paths: filename = os.path.basename(file_path) items = filename.split('-') if len(items) < 2: continue ctype = items[1] if filter_ctype and filter_ctype != ctype: continue dist = '' arch = '' if len(items) > 2: dist = items[2] if filter_dist and dist and filter_dist != dist and dist != ALL_DIST: continue if len(items) > 3: arch = items[3] if filter_arch and arch and filter_arch != arch and arch != ALL_ARCH: continue versions = Component.get_versions(file_path) component = Component(self.verbose, versions, ctype, dist, arch) components.append(component) if self.verbose and re.search("stage=load", self.verbose): component.print(file_path) def load_from_target(self, image_path): self.module_path=image_path post_versions = os.path.join(image_path, 'post-versions') if os.path.exists(post_versions): self.load(post_versions) self.name = os.path.basename(image_path) if self.verbose and re.search("stage=post", self.verbose): self.print(post_versions) pre_versions = os.path.join(image_path, 'pre-versions') if os.path.exists(pre_versions): pre_module = VersionModule(self.verbose) pre_module.load(pre_versions) if self.verbose and re.search("stage=pre", self.verbose): pre_module.print(pre_versions) self.subtract(pre_module) else: self.load(image_path) def dump(self, module_path, config=False, priority=999): version_file_pattern = os.path.join(module_path, VERSION_PREFIX + '*') for filename in glob.glob(version_file_pattern): os.remove(filename) for component in self.components: component.dump_to_path(module_path, config, priority) def print(self, module_path): if self.verbose is None: return if re.search("cmod=", self.verbose) \ and not re.search(self.verbose, "cmod=all".format(self.name)) \ and not re.search(self.verbose, "cmod={}".format(self.name)): return for component in self.components: component.print(module_path) def filter(self, ctypes=[]): if 'all' in ctypes: return self components = [] for component in self.components: if component.ctype in ctypes: components.append(component) self.components = components def clean_info(self, clean_dist=True, clean_arch=True, force=False): for component in self.components: component.clean_info(clean_dist=clean_dist, clean_arch=clean_arch, force=force) def clone(self, ctypes=None, exclude_ctypes=None): components = [] for component in self.components: if exclude_ctypes and component.ctype in exclude_ctypes: continue if ctypes and component.ctype not in ctypes: continue components.append(component.clone()) return VersionModule(self.verbose, self.name, components) def is_slave_module(self): return self.name.startswith('sonic-slave-') # Do not inherit the version from the default module def is_individule_version(self): return self.is_slave_module() and SLAVE_INDIVIDULE_VERSION @classmethod def is_aggregatable_module(cls, module_name): if module_name.startswith('sonic-slave-'): return False if module_name.startswith('build-sonic-slave-'): return False if module_name == DEFAULT_MODULE: return False if module_name == 'host-image' or module_name == 'host-base-image': return False return True @classmethod def get_module_path_by_name(cls, source_path, module_name): common_modules = ['default', 'host-image', 'host-base-image'] if module_name in common_modules: return os.path.join(source_path, 'files/build/versions', module_name) if module_name.startswith('build-sonic-slave-'): return os.path.join(source_path, 'files/build/versions/build', module_name) return os.path.join(source_path, 'files/build/versions/dockers', module_name) def __repr__(self): return repr(self.name) class VersionBuild: ''' The VersionBuild consists of multiple version modules. ''' def __init__(self, verbose=None, target_path="./target", source_path='.'): self.target_path = target_path self.source_path = source_path self.verbose = verbose self.modules = {} def load_from_target(self): dockers_path = os.path.join(self.target_path, 'versions/dockers') build_path = os.path.join(self.target_path, 'versions/build') default_path = os.path.join(self.target_path, 'versions/default') modules = {} self.modules = modules file_paths = glob.glob(dockers_path + '/*') file_paths += glob.glob(build_path + '/build-*') file_paths += glob.glob(default_path) file_paths.append(os.path.join(self.target_path, 'versions/host-image')) file_paths.append(os.path.join(self.target_path, 'versions/host-base-image')) for file_path in file_paths: if not os.path.isdir(file_path): continue module = VersionModule(self.verbose) module.load_from_target(file_path) if self.verbose and re.search("stage=tmodname", self.verbose): print("Target modname={}, path={}".format(module.name, file_path)) module.print(file_path) modules[module.name] = module self._merge_dgb_modules() def load_from_source(self): # Load default versions and host image versions versions_path = os.path.join(self.source_path, 'files/build/versions') dockers_path = os.path.join(versions_path, "dockers") build_path = os.path.join(versions_path, "build") paths = [os.path.join(versions_path, 'default')] paths += glob.glob(versions_path + '/host-*') paths += glob.glob(dockers_path + '/*') paths += glob.glob(build_path + '/*') modules = {} self.modules = modules for image_path in paths: module = VersionModule(self.verbose) module.load(image_path) if self.verbose and re.search("stage=smodname", self.verbose): print("Source modname={}, path={}".format(module.name, image_path)) module.print(image_path) modules[module.name] = module def overwrite(self, build, for_all_dist=False, for_all_arch=False): for target_module in build.modules.values(): module = self.modules.get(target_module.name, None) tmp_module = target_module.clone() tmp_module.clean_info(for_all_dist, for_all_arch) if module: module.overwrite(tmp_module, for_all_dist=for_all_dist, for_all_arch=for_all_arch) else: self.modules[target_module.name] = tmp_module def dump(self): for module in self.modules.values(): module_path = self.get_module_path(module) module.dump(module_path) def print(self, message=None): if self.verbose is None: return if message is not None: print("[============={}===========]".format(message)) for module in [ self.modules[x] for x in (sorted(self.modules, key = lambda x : x)) ]: module.print(module.module_path) def subtract(self, default_module): none_aggregatable_module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) for module in self.modules.values(): if module.name == DEFAULT_MODULE: continue if module.name == 'host-base-image': continue if module.is_individule_version(): continue tmp_module = default_module if not module.is_aggregatable_module(module.name): tmp_module = none_aggregatable_module module.subtract(tmp_module) def freeze(self, rebuild=False, for_all_dist=False, for_all_arch=False, ctypes=['all']): if rebuild: self.load_from_target() self.filter(ctypes=ctypes) default_module = self.get_default_module() self._clean_component_info() self.subtract(default_module) self.modules[DEFAULT_MODULE] = default_module self.dump() return self.load_from_source() if self.verbose and re.search("stage=init", self.verbose): self.print("Initial Source") default_module = self.modules.get(DEFAULT_MODULE, None) if self.verbose and re.search("stage=init", self.verbose): default_module.print("Default Module") target_build = VersionBuild(self.verbose, self.target_path, self.source_path) target_build.load_from_target() target_build.filter(ctypes=ctypes) if self.verbose and re.search("stage=init", self.verbose): target_build.print("Initial Target") if not default_module: raise Exception("The default versions does not exist") for module in [ target_build.modules[x] for x in (sorted(target_build.modules, key = lambda x : x)) ] : if module.is_individule_version(): continue tmp_module = module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) default_module.overwrite(tmp_module, for_all_dist=True, for_all_arch=True) if self.verbose and re.search("stage=tmp", self.verbose): default_module.print("TMP DEFAULT MODULE") target_build.subtract(default_module) if self.verbose and re.search("stage=tmp", self.verbose): target_build.print("After Subtract Target") self.print("After Subtract Source") self.overwrite(target_build, for_all_dist=for_all_dist, for_all_arch=for_all_arch) if self.verbose and re.search("stage=add", self.verbose): self.print("After Merge") if not self.verbose or not re.search("dryrun", self.verbose): self.dump() def filter(self, ctypes=[]): for module in self.modules.values(): module.filter(ctypes=ctypes) def get_default_module(self): if DEFAULT_MODULE in self.modules: return self.modules[DEFAULT_MODULE] ctypes = self.get_component_types() dists = self.get_dists() components = [] for ctype in ctypes: if ctype == 'deb': for dist in dists: versions = self._get_versions(ctype, dist) common_versions = self._get_common_versions(versions) component = Component(self.verbose, common_versions, ctype, dist) components.append(component) else: versions = self._get_versions(ctype) common_versions = self._get_common_versions(versions) component = Component(self.verbose, common_versions, ctype) components.append(component) return VersionModule(self.verbose, DEFAULT_MODULE, components) def get_aggregatable_modules(self): modules = {} for module_name in self.modules: if not VersionModule.is_aggregatable_module(module_name): continue module = self.modules[module_name] modules[module_name] = module return modules def get_components(self): components = [] for module_name in self.modules: module = self.modules[module_name] for component in module.components: components.append(component) return components def get_component_types(self): ctypes = [] for module_name in self.modules: module = self.modules[module_name] for component in module.components: if component.ctype not in ctypes: ctypes.append(component.ctype) return ctypes def get_dists(self): dists = [] components = self.get_components() for component in components: if component.dist not in dists: dists.append(component.dist) return dists def get_archs(self): archs = [] components = self.get_components() for component in components: if component.arch not in archs: archs.append(component.arch) return archs def get_module_path(self, module): return self.get_module_path_by_name(module.name) def get_module_path_by_name(self, module_name): return VersionModule.get_module_path_by_name(self.source_path, module_name) def _merge_dgb_modules(self): dbg_modules = [] for module_name in self.modules: if not module_name.endswith('-dbg'): continue dbg_modules.append(module_name) base_module_name = module_name[:-4] if base_module_name not in self.modules: raise Exception('The Module {0} not found'.format(base_module_name)) base_module = self.modules[base_module_name] dbg_module = self.modules[module_name] base_module.overwrite(dbg_module) for module_name in dbg_modules: del self.modules[module_name] def _clean_component_info(self, clean_dist=True, clean_arch=True): for module in self.modules.values(): module.clean_info(clean_dist, clean_arch) def _get_versions(self, ctype, dist=None, arch=None): versions = {} modules = self.get_aggregatable_modules() for module_name in self.modules: if module_name not in modules: temp_module = self.modules[module_name].clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) modules[module_name] = temp_module for module in modules.values(): for component in module.components: if ctype != component.ctype: continue if dist and dist != component.dist: continue if arch and arch != component.arch: continue for package in component.versions: version = component.versions[package] package_versions = versions.get(package, []) if version not in package_versions: package_versions.append(version) versions[package] = package_versions return versions def _get_common_versions(self, versions): common_versions = {} for package in versions: package_versions = versions[package] if len(package_versions) == 1: common_versions[package] = package_versions[0] return common_versions class VersionManagerCommands: def __init__(self): usage = 'version_manager.py []\n\n' usage = usage + 'The most commonly used commands are:\n' usage = usage + ' freeze Freeze the version files\n' usage = usage + ' generate Generate the version files\n' usage = usage + ' merge Merge the version files' parser = argparse.ArgumentParser(description='Version manager', usage=usage) parser.add_argument('command', help='Subcommand to run') args = parser.parse_args(sys.argv[1:2]) if not hasattr(self, args.command): print('Unrecognized command: {0}'.format(args.command)) parser.print_help() exit(1) getattr(self, args.command)() def freeze(self): parser = argparse.ArgumentParser(description = 'Freeze the version files') parser.add_argument('-t', '--target_path', default='./target', help='target path') parser.add_argument('-s', '--source_path', default='.', help='source path') # store_true which implies default=False parser.add_argument('-r', '--rebuild', action='store_true', help='rebuild all versions') parser.add_argument('-d', '--for_all_dist', action='store_true', help='apply the versions for all distributions') parser.add_argument('-a', '--for_all_arch', action='store_true', help='apply the versions for all architectures') parser.add_argument('-c', '--ctypes', default='all', help='component types to freeze') parser.add_argument('-v', '--verbose', default=None, help="verbose mode") args = parser.parse_args(sys.argv[2:]) ctypes = args.ctypes.split(',') if len(ctypes) == 0: ctypes = ['all'] build = VersionBuild(verbose=args.verbose, target_path=args.target_path, source_path=args.source_path) build.freeze(rebuild=args.rebuild, for_all_dist=args.for_all_dist, for_all_arch=args.for_all_arch, ctypes=ctypes) def merge(self): parser = argparse.ArgumentParser(description = 'Merge the version files') parser.add_argument('-t', '--target_path', required=True, help='target path to save the merged version files') parser.add_argument('-m', '--module_path', default=None, help='merge path, use the target path if not specified') parser.add_argument('-b', '--base_path', required=True, help='base path, merge to the module path') parser.add_argument('-e', '--exclude_module_path', default=None, help='exclude module path') parser.add_argument('-i', '--include_module_path', default=None, help='include module path') parser.add_argument('-v', '--verbose', default=None, help="verbose mode") args = parser.parse_args(sys.argv[2:]) module_path = args.module_path if not module_path: module_path = args.target_path if not os.path.exists(module_path): print('The module path {0} does not exist'.format(module_path)) if not os.path.exists(args.target_path): os.makedirs(args.target_path) module = VersionModule(args.verbose) module.load(module_path) base_module = VersionModule(args.verbose) base_module.load(args.base_path) module.overwrite(base_module) if args.exclude_module_path: exclude_module = VersionModule(args.verbose) exclude_module.load(args.exclude_module_path) module.subtract(exclude_module) if args.include_module_path: include_module = VersionModule(args.verbose) include_module.load(args.include_module_path) if args.verbose: include_module.print(args.include_module_path) include_module.overwrite(module) module.overwrite(include_module) module.dump(args.target_path) def generate(self): parser = argparse.ArgumentParser(description = 'Generate the version files') parser.add_argument('-t', '--target_path', required=True, help='target path to generate the version lock files') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-n', '--module_name', help="module name, such as docker-lldp, sonic-slave-buster, etc") group.add_argument('-m', '--module_path', help="module apth, such as files/docker/versions/dockers/docker-lldp, files/docker/versions/dockers/sonic-slave-buster, etc") parser.add_argument('-s', '--source_path', default='.', help='source path') parser.add_argument('-d', '--distribution', required=True, help="distribution") parser.add_argument('-a', '--architecture', required=True, help="architecture") parser.add_argument('-p', '--priority', default=999, help="priority of the debian apt preference") parser.add_argument('-v', '--verbose', default=None, help="verbose mode") args = parser.parse_args(sys.argv[2:]) module_path = args.module_path if not module_path: module_path = VersionModule.get_module_path_by_name(args.source_path, args.module_name) if not os.path.exists(args.target_path): os.makedirs(args.target_path) module = VersionModule(args.verbose) module.load(module_path, filter_dist=args.distribution, filter_arch=args.architecture) config = module.get_config_module(args.source_path, args.distribution, args.architecture) if args.verbose: config.print(args.source_path) config.clean_info(force=True) config.dump(args.target_path, config=True, priority=args.priority) if __name__ == "__main__": VersionManagerCommands() """ Dry run examples: scripts/versions_manager.py freeze -v 'dryrun|cmod=docker-config-engine-stretch|cfile=versions-py2|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp' scripts/versions_manager.py freeze -v 'dryrun|cmod=default|cfile=versions-docker|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp' """