2020-12-21 01:31:10 -06:00
#!/usr/bin/python3
import argparse
import glob
import os
import sys
2022-12-11 19:20:56 -06:00
import re
2020-12-21 01:31:10 -06:00
ALL_DIST = ' all '
ALL_ARCH = ' all '
DEFAULT_MODULE = ' default '
DEFAULT_VERSION_PATH = ' files/build/versions '
VERSION_PREFIX = " versions- "
VERSION_DEB_PREFERENCE = ' 01-versions-deb '
DEFAULT_OVERWRITE_COMPONENTS = [ ' deb ' , ' py2 ' , ' py3 ' ]
SLAVE_INDIVIDULE_VERSION = False
class Component :
'''
The component consists of mutiple packages
ctype - - Component Type , such as deb , py2 , etc
dist - - Distribution , such as stretch , buster , etc
arch - - Architectrue , such as amd64 , arm64 , etc
2021-03-30 10:34:25 -05:00
2020-12-21 01:31:10 -06:00
'''
2022-12-11 19:20:56 -06:00
def __init__ ( self , verbose = None , versions = { } , ctype = " deb " , dist = ALL_DIST , arch = ALL_ARCH ) :
2020-12-21 01:31:10 -06:00
self . versions = versions
self . ctype = ctype
if not dist :
dist = ALL_DIST
if not arch :
arch = ALL_ARCH
self . dist = dist
self . arch = arch
2022-12-11 19:20:56 -06:00
self . verbose = verbose
2020-12-21 01:31:10 -06:00
@classmethod
def get_versions ( cls , version_file ) :
result = { }
if not os . path . exists ( version_file ) :
return result
with open ( version_file ) as fp :
for line in fp . readlines ( ) :
offset = line . rfind ( ' == ' )
if offset > 0 :
package = line [ : offset ] . strip ( )
2021-03-30 10:34:25 -05:00
if ' py2 ' in version_file . lower ( ) or ' py3 ' in version_file . lower ( ) :
package = package . lower ( )
2020-12-21 01:31:10 -06:00
version = line [ offset + 2 : ] . strip ( )
result [ package ] = version
return result
def clone ( self ) :
2022-12-11 19:20:56 -06:00
return Component ( self . verbose , self . versions . copy ( ) , self . ctype , self . dist , self . arch )
2020-12-21 01:31:10 -06:00
def merge ( self , versions , overwritten = True ) :
for package in versions :
if overwritten or package not in self . versions :
self . versions [ package ] = versions [ package ]
def subtract ( self , versions ) :
for package in versions :
if package in self . versions and self . versions [ package ] == versions [ package ] :
del self . versions [ package ]
def dump ( self , config = False , priority = 999 ) :
result = [ ]
for package in sorted ( self . versions . keys ( ) , key = str . casefold ) :
if config and self . ctype == ' deb ' :
lines = ' Package: {0} \n Pin: version {1} \n Pin-Priority: {2} \n \n ' . format ( package , self . versions [ package ] , priority )
result . append ( lines )
else :
result . append ( ' {0} == {1} ' . format ( package , self . versions [ package ] ) )
2022-12-11 19:20:56 -06:00
return " \n " . join ( result ) + ' \n '
2020-12-21 01:31:10 -06:00
def dump_to_file ( self , version_file , config = False , priority = 999 ) :
if len ( self . versions ) < = 0 :
return
with open ( version_file , ' w ' ) as f :
f . write ( self . dump ( config , priority ) )
def dump_to_path ( self , file_path , config = False , priority = 999 ) :
if len ( self . versions ) < = 0 :
return
if not os . path . exists ( file_path ) :
os . makedirs ( file_path )
filename = self . get_filename ( )
if config and self . ctype == ' deb ' :
none_config_file_path = os . path . join ( file_path , filename )
self . dump_to_file ( none_config_file_path , False , priority )
2021-03-30 10:34:25 -05:00
filename = VERSION_DEB_PREFERENCE
2020-12-21 01:31:10 -06:00
file_path = os . path . join ( file_path , filename )
self . dump_to_file ( file_path , config , priority )
2022-12-11 19:20:56 -06:00
def print ( self , file_path ) :
if len ( self . versions ) < = 0 :
return
if self . verbose is None :
return
filename = self . get_filename ( )
file_path = os . path . join ( file_path , filename )
if self . verbose and re . search ( " cfile= " , self . verbose ) \
and not re . search ( self . verbose , " cfile=all " . format ( filename ) ) \
and not re . search ( self . verbose , " cfile= {} " . format ( filename ) ) :
return
print ( " VERSION : {} " . format ( file_path ) )
for package in sorted ( self . versions . keys ( ) , key = str . casefold ) :
if self . verbose and re . search ( " ctype= " , self . verbose ) \
and not re . search ( " ctype=all " . format ( self . ctype ) , self . verbose ) \
and not re . search ( " ctype= {} " . format ( self . ctype ) , self . verbose ) :
continue
if self . verbose and re . search ( " cname= " , self . verbose ) \
and not re . search ( self . verbose , " cname=all " . format ( package ) ) \
and not re . search ( self . verbose , " cname= {} " . format ( package ) ) :
continue
if self . verbose and re . search ( " cver= " , self . verbose ) \
and not re . search ( self . verbose , " cver=all " . format ( self . versions [ package ] ) ) \
and not re . search ( self . verbose , " cver= {} " . format ( self . versions [ package ] ) ) :
continue
print ( ' {0} == {1} ' . format ( package , self . versions [ package ] ) )
2020-12-21 01:31:10 -06:00
# Check if the self component can be overwritten by the input component
def check_overwritable ( self , component , for_all_dist = False , for_all_arch = False ) :
if self . ctype != component . ctype :
return False
if self . dist != component . dist and not ( for_all_dist and self . dist == ALL_DIST ) :
return False
if self . arch != component . arch and not ( for_all_arch and self . arch == ALL_ARCH ) :
return False
return True
# Check if the self component can inherit the package versions from the input component
def check_inheritable ( self , component ) :
if self . ctype != component . ctype :
return False
if self . dist != component . dist and component . dist != ALL_DIST :
return False
if self . arch != component . arch and component . arch != ALL_ARCH :
return False
return True
'''
Get the file name
The file name format : versions - { ctype } - { dist } - { arch }
If { arch } is all , then the file name format : versions - { ctype } - { dist }
if { arch } is all and { dist } is all , then the file name format : versions - { ctype }
'''
def get_filename ( self ) :
filename = VERSION_PREFIX + self . ctype
dist = self . dist
if self . arch and self . arch != ALL_ARCH :
if not dist :
dist = ALL_DIST
return filename + ' - ' + dist + ' - ' + self . arch
if dist and self . dist != ALL_DIST :
filename = filename + ' - ' + dist
return filename
def get_order_keys ( self ) :
dist = self . dist
if not dist or dist == ALL_DIST :
dist = ' '
arch = self . arch
if not arch or arch == ALL_ARCH :
arch = ' '
return ( self . ctype , dist , arch )
def clean_info ( self , clean_dist = True , clean_arch = True , force = False ) :
if clean_dist :
if force or self . ctype != ' deb ' :
self . dist = ALL_DIST
if clean_arch :
self . arch = ALL_ARCH
class VersionModule :
'''
The version module represents a build target , such as docker image , host image , consists of multiple components .
name - - The name of the image , such as sonic - slave - buster , docker - lldp , etc
'''
2022-12-11 19:20:56 -06:00
def __init__ ( self , verbose = None , name = None , components = None ) :
2020-12-21 01:31:10 -06:00
self . name = name
self . components = components
2022-12-11 19:20:56 -06:00
self . module_path = " "
self . verbose = verbose
2020-12-21 01:31:10 -06:00
# Overwrite the docker/host image/base image versions
def overwrite ( self , module , for_all_dist = False , for_all_arch = False ) :
# Overwrite from generic one to detail one
# For examples: versions-deb overwrtten by versions-deb-buster, and versions-deb-buster overwritten by versions-deb-buster-amd64
components = sorted ( module . components , key = lambda x : x . get_order_keys ( ) )
for merge_component in components :
merged = False
for component in self . components :
if component . check_overwritable ( merge_component , for_all_dist = for_all_dist , for_all_arch = for_all_arch ) :
component . merge ( merge_component . versions , True )
merged = True
if not merged :
tmp_component = merge_component . clone ( )
tmp_component . clean_info ( clean_dist = for_all_dist , clean_arch = for_all_arch )
self . components . append ( tmp_component )
self . adjust ( )
2022-03-28 23:34:46 -05:00
def get_config_module ( self , source_path , dist , arch ) :
2020-12-21 01:31:10 -06:00
if self . is_individule_version ( ) :
return self
2022-03-28 23:34:46 -05:00
default_module_path = VersionModule . get_module_path_by_name ( source_path , DEFAULT_MODULE )
default_module = VersionModule ( )
default_module . load ( default_module_path , filter_dist = dist , filter_arch = arch )
2020-12-21 01:31:10 -06:00
module = default_module
2022-03-28 23:34:46 -05:00
if self . name == ' host-image ' :
base_module_path = VersionModule . get_module_path_by_name ( source_path , ' host-base-image ' )
base_module = VersionModule ( )
base_module . load ( base_module_path , filter_dist = dist , filter_arch = arch )
module = default_module . clone ( exclude_ctypes = DEFAULT_OVERWRITE_COMPONENTS )
module . overwrite ( base_module , True , True )
elif not self . is_aggregatable_module ( self . name ) :
2020-12-21 01:31:10 -06:00
module = default_module . clone ( exclude_ctypes = DEFAULT_OVERWRITE_COMPONENTS )
return self . _get_config_module ( module , dist , arch )
2022-12-11 19:20:56 -06:00
#Merge the default with specific version
2020-12-21 01:31:10 -06:00
def _get_config_module ( self , default_module , dist , arch ) :
module = default_module . clone ( )
default_ctype_components = module . _get_components_per_ctypes ( )
module . overwrite ( self )
config_components = [ ]
ctype_components = module . _get_components_per_ctypes ( )
for ctype in default_ctype_components :
if ctype not in ctype_components :
ctype_components [ ctype ] = [ ]
for components in ctype_components . values ( ) :
2021-01-05 05:05:13 -06:00
if len ( components ) == 0 :
continue
2020-12-21 01:31:10 -06:00
config_component = self . _get_config_for_ctype ( components , dist , arch )
config_components . append ( config_component )
2022-12-11 19:20:56 -06:00
config_module = VersionModule ( self . verbose , self . name , config_components )
2020-12-21 01:31:10 -06:00
return config_module
def _get_config_for_ctype ( self , components , dist , arch ) :
2022-12-11 19:20:56 -06:00
result = Component ( self . verbose , { } , components [ 0 ] . ctype , dist , arch )
2020-12-21 01:31:10 -06:00
for component in sorted ( components , key = lambda x : x . get_order_keys ( ) ) :
if result . check_inheritable ( component ) :
result . merge ( component . versions , True )
return result
def subtract ( self , default_module ) :
module = self . clone ( )
result = [ ]
ctype_components = module . _get_components_per_ctypes ( )
for ctype in ctype_components :
components = ctype_components [ ctype ]
components = sorted ( components , key = lambda x : x . get_order_keys ( ) )
for i in range ( 0 , len ( components ) ) :
component = components [ i ]
2022-12-11 19:20:56 -06:00
base_module = VersionModule ( self . verbose , self . name , components [ 0 : i ] )
2020-12-21 01:31:10 -06:00
config_module = base_module . _get_config_module ( default_module , component . dist , component . arch )
config_components = config_module . _get_components_by_ctype ( ctype )
if len ( config_components ) > 0 :
config_component = config_components [ 0 ]
component . subtract ( config_component . versions )
if len ( component . versions ) :
result . append ( component )
self . components = result
def adjust ( self ) :
result_components = [ ]
ctype_components = self . _get_components_per_ctypes ( )
for components in ctype_components . values ( ) :
result_components + = self . _adjust_components_for_ctype ( components )
self . components = result_components
def _get_components_by_ctype ( self , ctype ) :
components = [ ]
for component in self . components :
if component . ctype == ctype :
components . append ( component )
return components
def _adjust_components_for_ctype ( self , components ) :
components = sorted ( components , key = lambda x : x . get_order_keys ( ) )
result = [ ]
for i in range ( 0 , len ( components ) ) :
component = components [ i ]
2022-12-11 19:20:56 -06:00
inheritable_component = Component ( self . verbose , { } , component . ctype )
2020-12-21 01:31:10 -06:00
for j in range ( 0 , i ) :
base_component = components [ j ]
if component . check_inheritable ( base_component ) :
inheritable_component . merge ( base_component . versions , True )
component . subtract ( inheritable_component . versions )
if len ( component . versions ) > 0 :
result . append ( component )
return result
def _get_components_per_ctypes ( self ) :
result = { }
for component in self . components :
components = result . get ( component . ctype , [ ] )
components . append ( component )
result [ component . ctype ] = components
return result
def load ( self , image_path , filter_ctype = None , filter_dist = None , filter_arch = None ) :
version_file_pattern = os . path . join ( image_path , VERSION_PREFIX ) + ' * '
file_paths = glob . glob ( version_file_pattern )
components = [ ]
self . name = os . path . basename ( image_path )
2022-12-11 19:20:56 -06:00
self . module_path = image_path
2020-12-21 01:31:10 -06:00
self . components = components
for file_path in file_paths :
filename = os . path . basename ( file_path )
items = filename . split ( ' - ' )
if len ( items ) < 2 :
continue
ctype = items [ 1 ]
if filter_ctype and filter_ctype != ctype :
continue
dist = ' '
arch = ' '
if len ( items ) > 2 :
dist = items [ 2 ]
2021-11-30 17:35:07 -06:00
if filter_dist and dist and filter_dist != dist and dist != ALL_DIST :
2020-12-21 01:31:10 -06:00
continue
if len ( items ) > 3 :
arch = items [ 3 ]
2021-11-30 17:35:07 -06:00
if filter_arch and arch and filter_arch != arch and arch != ALL_ARCH :
2020-12-21 01:31:10 -06:00
continue
versions = Component . get_versions ( file_path )
2022-12-11 19:20:56 -06:00
component = Component ( self . verbose , versions , ctype , dist , arch )
2020-12-21 01:31:10 -06:00
components . append ( component )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=load " , self . verbose ) :
component . print ( file_path )
2020-12-21 01:31:10 -06:00
def load_from_target ( self , image_path ) :
2022-12-11 19:20:56 -06:00
self . module_path = image_path
2020-12-21 01:31:10 -06:00
post_versions = os . path . join ( image_path , ' post-versions ' )
if os . path . exists ( post_versions ) :
self . load ( post_versions )
self . name = os . path . basename ( image_path )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=post " , self . verbose ) :
self . print ( post_versions )
2020-12-21 01:31:10 -06:00
pre_versions = os . path . join ( image_path , ' pre-versions ' )
if os . path . exists ( pre_versions ) :
2022-12-11 19:20:56 -06:00
pre_module = VersionModule ( self . verbose )
2020-12-21 01:31:10 -06:00
pre_module . load ( pre_versions )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=pre " , self . verbose ) :
pre_module . print ( pre_versions )
2020-12-21 01:31:10 -06:00
self . subtract ( pre_module )
else :
self . load ( image_path )
def dump ( self , module_path , config = False , priority = 999 ) :
version_file_pattern = os . path . join ( module_path , VERSION_PREFIX + ' * ' )
for filename in glob . glob ( version_file_pattern ) :
os . remove ( filename )
for component in self . components :
component . dump_to_path ( module_path , config , priority )
2022-12-11 19:20:56 -06:00
def print ( self , module_path ) :
if self . verbose is None :
return
if re . search ( " cmod= " , self . verbose ) \
and not re . search ( self . verbose , " cmod=all " . format ( self . name ) ) \
and not re . search ( self . verbose , " cmod= {} " . format ( self . name ) ) :
return
for component in self . components :
component . print ( module_path )
2020-12-21 01:31:10 -06:00
def filter ( self , ctypes = [ ] ) :
if ' all ' in ctypes :
return self
components = [ ]
for component in self . components :
if component . ctype in ctypes :
components . append ( component )
self . components = components
def clean_info ( self , clean_dist = True , clean_arch = True , force = False ) :
for component in self . components :
component . clean_info ( clean_dist = clean_dist , clean_arch = clean_arch , force = force )
def clone ( self , ctypes = None , exclude_ctypes = None ) :
components = [ ]
for component in self . components :
if exclude_ctypes and component . ctype in exclude_ctypes :
continue
if ctypes and component . ctype not in ctypes :
continue
components . append ( component . clone ( ) )
2022-12-11 19:20:56 -06:00
return VersionModule ( self . verbose , self . name , components )
2020-12-21 01:31:10 -06:00
def is_slave_module ( self ) :
return self . name . startswith ( ' sonic-slave- ' )
# Do not inherit the version from the default module
def is_individule_version ( self ) :
return self . is_slave_module ( ) and SLAVE_INDIVIDULE_VERSION
@classmethod
def is_aggregatable_module ( cls , module_name ) :
if module_name . startswith ( ' sonic-slave- ' ) :
return False
if module_name . startswith ( ' build-sonic-slave- ' ) :
return False
if module_name == DEFAULT_MODULE :
return False
if module_name == ' host-image ' or module_name == ' host-base-image ' :
return False
return True
@classmethod
def get_module_path_by_name ( cls , source_path , module_name ) :
common_modules = [ ' default ' , ' host-image ' , ' host-base-image ' ]
if module_name in common_modules :
return os . path . join ( source_path , ' files/build/versions ' , module_name )
if module_name . startswith ( ' build-sonic-slave- ' ) :
return os . path . join ( source_path , ' files/build/versions/build ' , module_name )
return os . path . join ( source_path , ' files/build/versions/dockers ' , module_name )
2022-12-11 19:20:56 -06:00
def __repr__ ( self ) :
return repr ( self . name )
2020-12-21 01:31:10 -06:00
class VersionBuild :
'''
The VersionBuild consists of multiple version modules .
'''
2022-12-11 19:20:56 -06:00
def __init__ ( self , verbose = None , target_path = " ./target " , source_path = ' . ' ) :
2020-12-21 01:31:10 -06:00
self . target_path = target_path
self . source_path = source_path
2022-12-11 19:20:56 -06:00
self . verbose = verbose
2020-12-21 01:31:10 -06:00
self . modules = { }
def load_from_target ( self ) :
dockers_path = os . path . join ( self . target_path , ' versions/dockers ' )
build_path = os . path . join ( self . target_path , ' versions/build ' )
2021-07-09 01:30:26 -05:00
default_path = os . path . join ( self . target_path , ' versions/default ' )
2020-12-21 01:31:10 -06:00
modules = { }
self . modules = modules
file_paths = glob . glob ( dockers_path + ' /* ' )
file_paths + = glob . glob ( build_path + ' /build-* ' )
2021-07-09 01:30:26 -05:00
file_paths + = glob . glob ( default_path )
2020-12-21 01:31:10 -06:00
file_paths . append ( os . path . join ( self . target_path , ' versions/host-image ' ) )
file_paths . append ( os . path . join ( self . target_path , ' versions/host-base-image ' ) )
for file_path in file_paths :
if not os . path . isdir ( file_path ) :
continue
2022-12-11 19:20:56 -06:00
module = VersionModule ( self . verbose )
2020-12-21 01:31:10 -06:00
module . load_from_target ( file_path )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=tmodname " , self . verbose ) :
print ( " Target modname= {} , path= {} " . format ( module . name , file_path ) )
module . print ( file_path )
2020-12-21 01:31:10 -06:00
modules [ module . name ] = module
self . _merge_dgb_modules ( )
def load_from_source ( self ) :
# Load default versions and host image versions
versions_path = os . path . join ( self . source_path , ' files/build/versions ' )
dockers_path = os . path . join ( versions_path , " dockers " )
build_path = os . path . join ( versions_path , " build " )
paths = [ os . path . join ( versions_path , ' default ' ) ]
paths + = glob . glob ( versions_path + ' /host-* ' )
paths + = glob . glob ( dockers_path + ' /* ' )
paths + = glob . glob ( build_path + ' /* ' )
modules = { }
self . modules = modules
for image_path in paths :
2022-12-11 19:20:56 -06:00
module = VersionModule ( self . verbose )
2020-12-21 01:31:10 -06:00
module . load ( image_path )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=smodname " , self . verbose ) :
print ( " Source modname= {} , path= {} " . format ( module . name , image_path ) )
module . print ( image_path )
2020-12-21 01:31:10 -06:00
modules [ module . name ] = module
def overwrite ( self , build , for_all_dist = False , for_all_arch = False ) :
for target_module in build . modules . values ( ) :
module = self . modules . get ( target_module . name , None )
tmp_module = target_module . clone ( )
tmp_module . clean_info ( for_all_dist , for_all_arch )
if module :
module . overwrite ( tmp_module , for_all_dist = for_all_dist , for_all_arch = for_all_arch )
else :
self . modules [ target_module . name ] = tmp_module
def dump ( self ) :
for module in self . modules . values ( ) :
module_path = self . get_module_path ( module )
module . dump ( module_path )
2022-12-11 19:20:56 -06:00
def print ( self , message = None ) :
if self . verbose is None :
return
if message is not None :
print ( " [============= {} ===========] " . format ( message ) )
for module in [ self . modules [ x ] for x in ( sorted ( self . modules , key = lambda x : x ) ) ] :
module . print ( module . module_path )
2020-12-21 01:31:10 -06:00
def subtract ( self , default_module ) :
none_aggregatable_module = default_module . clone ( exclude_ctypes = DEFAULT_OVERWRITE_COMPONENTS )
for module in self . modules . values ( ) :
if module . name == DEFAULT_MODULE :
continue
if module . name == ' host-base-image ' :
continue
if module . is_individule_version ( ) :
continue
tmp_module = default_module
if not module . is_aggregatable_module ( module . name ) :
tmp_module = none_aggregatable_module
module . subtract ( tmp_module )
def freeze ( self , rebuild = False , for_all_dist = False , for_all_arch = False , ctypes = [ ' all ' ] ) :
if rebuild :
self . load_from_target ( )
self . filter ( ctypes = ctypes )
default_module = self . get_default_module ( )
self . _clean_component_info ( )
self . subtract ( default_module )
self . modules [ DEFAULT_MODULE ] = default_module
self . dump ( )
return
self . load_from_source ( )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=init " , self . verbose ) :
self . print ( " Initial Source " )
2020-12-21 01:31:10 -06:00
default_module = self . modules . get ( DEFAULT_MODULE , None )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=init " , self . verbose ) :
default_module . print ( " Default Module " )
target_build = VersionBuild ( self . verbose , self . target_path , self . source_path )
2020-12-21 01:31:10 -06:00
target_build . load_from_target ( )
target_build . filter ( ctypes = ctypes )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=init " , self . verbose ) :
target_build . print ( " Initial Target " )
2020-12-21 01:31:10 -06:00
if not default_module :
raise Exception ( " The default versions does not exist " )
2022-12-11 19:20:56 -06:00
for module in [ target_build . modules [ x ] for x in ( sorted ( target_build . modules , key = lambda x : x ) ) ] :
2020-12-21 01:31:10 -06:00
if module . is_individule_version ( ) :
continue
tmp_module = module . clone ( exclude_ctypes = DEFAULT_OVERWRITE_COMPONENTS )
default_module . overwrite ( tmp_module , for_all_dist = True , for_all_arch = True )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=tmp " , self . verbose ) :
default_module . print ( " TMP DEFAULT MODULE " )
2020-12-21 01:31:10 -06:00
target_build . subtract ( default_module )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=tmp " , self . verbose ) :
target_build . print ( " After Subtract Target " )
self . print ( " After Subtract Source " )
2020-12-21 01:31:10 -06:00
self . overwrite ( target_build , for_all_dist = for_all_dist , for_all_arch = for_all_arch )
2022-12-11 19:20:56 -06:00
if self . verbose and re . search ( " stage=add " , self . verbose ) :
self . print ( " After Merge " )
if not self . verbose or not re . search ( " dryrun " , self . verbose ) :
self . dump ( )
2020-12-21 01:31:10 -06:00
def filter ( self , ctypes = [ ] ) :
for module in self . modules . values ( ) :
module . filter ( ctypes = ctypes )
def get_default_module ( self ) :
if DEFAULT_MODULE in self . modules :
return self . modules [ DEFAULT_MODULE ]
ctypes = self . get_component_types ( )
dists = self . get_dists ( )
components = [ ]
for ctype in ctypes :
if ctype == ' deb ' :
for dist in dists :
versions = self . _get_versions ( ctype , dist )
common_versions = self . _get_common_versions ( versions )
2022-12-11 19:20:56 -06:00
component = Component ( self . verbose , common_versions , ctype , dist )
2020-12-21 01:31:10 -06:00
components . append ( component )
else :
versions = self . _get_versions ( ctype )
common_versions = self . _get_common_versions ( versions )
2022-12-11 19:20:56 -06:00
component = Component ( self . verbose , common_versions , ctype )
2020-12-21 01:31:10 -06:00
components . append ( component )
2022-12-11 19:20:56 -06:00
return VersionModule ( self . verbose , DEFAULT_MODULE , components )
2020-12-21 01:31:10 -06:00
def get_aggregatable_modules ( self ) :
modules = { }
for module_name in self . modules :
if not VersionModule . is_aggregatable_module ( module_name ) :
continue
module = self . modules [ module_name ]
modules [ module_name ] = module
return modules
def get_components ( self ) :
components = [ ]
for module_name in self . modules :
module = self . modules [ module_name ]
for component in module . components :
components . append ( component )
return components
def get_component_types ( self ) :
ctypes = [ ]
for module_name in self . modules :
module = self . modules [ module_name ]
for component in module . components :
if component . ctype not in ctypes :
ctypes . append ( component . ctype )
return ctypes
def get_dists ( self ) :
dists = [ ]
components = self . get_components ( )
for component in components :
if component . dist not in dists :
dists . append ( component . dist )
return dists
def get_archs ( self ) :
archs = [ ]
components = self . get_components ( )
for component in components :
if component . arch not in archs :
archs . append ( component . arch )
return archs
def get_module_path ( self , module ) :
return self . get_module_path_by_name ( module . name )
def get_module_path_by_name ( self , module_name ) :
return VersionModule . get_module_path_by_name ( self . source_path , module_name )
def _merge_dgb_modules ( self ) :
dbg_modules = [ ]
for module_name in self . modules :
if not module_name . endswith ( ' -dbg ' ) :
continue
dbg_modules . append ( module_name )
base_module_name = module_name [ : - 4 ]
if base_module_name not in self . modules :
raise Exception ( ' The Module {0} not found ' . format ( base_module_name ) )
base_module = self . modules [ base_module_name ]
dbg_module = self . modules [ module_name ]
base_module . overwrite ( dbg_module )
for module_name in dbg_modules :
del self . modules [ module_name ]
def _clean_component_info ( self , clean_dist = True , clean_arch = True ) :
for module in self . modules . values ( ) :
module . clean_info ( clean_dist , clean_arch )
def _get_versions ( self , ctype , dist = None , arch = None ) :
versions = { }
modules = self . get_aggregatable_modules ( )
for module_name in self . modules :
if module_name not in modules :
temp_module = self . modules [ module_name ] . clone ( exclude_ctypes = DEFAULT_OVERWRITE_COMPONENTS )
modules [ module_name ] = temp_module
for module in modules . values ( ) :
for component in module . components :
if ctype != component . ctype :
continue
if dist and dist != component . dist :
continue
if arch and arch != component . arch :
continue
for package in component . versions :
version = component . versions [ package ]
package_versions = versions . get ( package , [ ] )
if version not in package_versions :
package_versions . append ( version )
versions [ package ] = package_versions
return versions
def _get_common_versions ( self , versions ) :
common_versions = { }
for package in versions :
package_versions = versions [ package ]
if len ( package_versions ) == 1 :
common_versions [ package ] = package_versions [ 0 ]
return common_versions
class VersionManagerCommands :
def __init__ ( self ) :
usage = ' version_manager.py <command> [<args>] \n \n '
usage = usage + ' The most commonly used commands are: \n '
usage = usage + ' freeze Freeze the version files \n '
usage = usage + ' generate Generate the version files \n '
usage = usage + ' merge Merge the version files '
parser = argparse . ArgumentParser ( description = ' Version manager ' , usage = usage )
parser . add_argument ( ' command ' , help = ' Subcommand to run ' )
args = parser . parse_args ( sys . argv [ 1 : 2 ] )
if not hasattr ( self , args . command ) :
print ( ' Unrecognized command: {0} ' . format ( args . command ) )
parser . print_help ( )
exit ( 1 )
getattr ( self , args . command ) ( )
def freeze ( self ) :
parser = argparse . ArgumentParser ( description = ' Freeze the version files ' )
parser . add_argument ( ' -t ' , ' --target_path ' , default = ' ./target ' , help = ' target path ' )
parser . add_argument ( ' -s ' , ' --source_path ' , default = ' . ' , help = ' source path ' )
# store_true which implies default=False
parser . add_argument ( ' -r ' , ' --rebuild ' , action = ' store_true ' , help = ' rebuild all versions ' )
parser . add_argument ( ' -d ' , ' --for_all_dist ' , action = ' store_true ' , help = ' apply the versions for all distributions ' )
parser . add_argument ( ' -a ' , ' --for_all_arch ' , action = ' store_true ' , help = ' apply the versions for all architectures ' )
parser . add_argument ( ' -c ' , ' --ctypes ' , default = ' all ' , help = ' component types to freeze ' )
2022-12-11 19:20:56 -06:00
parser . add_argument ( ' -v ' , ' --verbose ' , default = None , help = " verbose mode " )
2020-12-21 01:31:10 -06:00
args = parser . parse_args ( sys . argv [ 2 : ] )
ctypes = args . ctypes . split ( ' , ' )
if len ( ctypes ) == 0 :
ctypes = [ ' all ' ]
2022-12-11 19:20:56 -06:00
build = VersionBuild ( verbose = args . verbose , target_path = args . target_path , source_path = args . source_path )
2020-12-21 01:31:10 -06:00
build . freeze ( rebuild = args . rebuild , for_all_dist = args . for_all_dist , for_all_arch = args . for_all_arch , ctypes = ctypes )
def merge ( self ) :
parser = argparse . ArgumentParser ( description = ' Merge the version files ' )
parser . add_argument ( ' -t ' , ' --target_path ' , required = True , help = ' target path to save the merged version files ' )
parser . add_argument ( ' -m ' , ' --module_path ' , default = None , help = ' merge path, use the target path if not specified ' )
parser . add_argument ( ' -b ' , ' --base_path ' , required = True , help = ' base path, merge to the module path ' )
parser . add_argument ( ' -e ' , ' --exclude_module_path ' , default = None , help = ' exclude module path ' )
2022-12-11 19:20:56 -06:00
parser . add_argument ( ' -i ' , ' --include_module_path ' , default = None , help = ' include module path ' )
parser . add_argument ( ' -v ' , ' --verbose ' , default = None , help = " verbose mode " )
2020-12-21 01:31:10 -06:00
args = parser . parse_args ( sys . argv [ 2 : ] )
module_path = args . module_path
if not module_path :
module_path = args . target_path
if not os . path . exists ( module_path ) :
print ( ' The module path {0} does not exist ' . format ( module_path ) )
if not os . path . exists ( args . target_path ) :
os . makedirs ( args . target_path )
2022-12-11 19:20:56 -06:00
module = VersionModule ( args . verbose )
2020-12-21 01:31:10 -06:00
module . load ( module_path )
2022-12-11 19:20:56 -06:00
base_module = VersionModule ( args . verbose )
2020-12-21 01:31:10 -06:00
base_module . load ( args . base_path )
module . overwrite ( base_module )
if args . exclude_module_path :
2022-12-11 19:20:56 -06:00
exclude_module = VersionModule ( args . verbose )
2020-12-21 01:31:10 -06:00
exclude_module . load ( args . exclude_module_path )
module . subtract ( exclude_module )
2022-12-11 19:20:56 -06:00
if args . include_module_path :
include_module = VersionModule ( args . verbose )
include_module . load ( args . include_module_path )
if args . verbose :
include_module . print ( args . include_module_path )
include_module . overwrite ( module )
module . overwrite ( include_module )
2020-12-21 01:31:10 -06:00
module . dump ( args . target_path )
def generate ( self ) :
parser = argparse . ArgumentParser ( description = ' Generate the version files ' )
parser . add_argument ( ' -t ' , ' --target_path ' , required = True , help = ' target path to generate the version lock files ' )
group = parser . add_mutually_exclusive_group ( required = True )
group . add_argument ( ' -n ' , ' --module_name ' , help = " module name, such as docker-lldp, sonic-slave-buster, etc " )
group . add_argument ( ' -m ' , ' --module_path ' , help = " module apth, such as files/docker/versions/dockers/docker-lldp, files/docker/versions/dockers/sonic-slave-buster, etc " )
parser . add_argument ( ' -s ' , ' --source_path ' , default = ' . ' , help = ' source path ' )
parser . add_argument ( ' -d ' , ' --distribution ' , required = True , help = " distribution " )
parser . add_argument ( ' -a ' , ' --architecture ' , required = True , help = " architecture " )
parser . add_argument ( ' -p ' , ' --priority ' , default = 999 , help = " priority of the debian apt preference " )
2022-12-11 19:20:56 -06:00
parser . add_argument ( ' -v ' , ' --verbose ' , default = None , help = " verbose mode " )
2020-12-21 01:31:10 -06:00
args = parser . parse_args ( sys . argv [ 2 : ] )
module_path = args . module_path
if not module_path :
module_path = VersionModule . get_module_path_by_name ( args . source_path , args . module_name )
if not os . path . exists ( args . target_path ) :
os . makedirs ( args . target_path )
2022-12-11 19:20:56 -06:00
module = VersionModule ( args . verbose )
2020-12-21 01:31:10 -06:00
module . load ( module_path , filter_dist = args . distribution , filter_arch = args . architecture )
2022-03-28 23:34:46 -05:00
config = module . get_config_module ( args . source_path , args . distribution , args . architecture )
2022-12-11 19:20:56 -06:00
if args . verbose :
config . print ( args . source_path )
2020-12-21 01:31:10 -06:00
config . clean_info ( force = True )
config . dump ( args . target_path , config = True , priority = args . priority )
if __name__ == " __main__ " :
VersionManagerCommands ( )
2022-12-11 19:20:56 -06:00
"""
Dry run examples :
scripts / versions_manager . py freeze - v ' dryrun|cmod=docker-config-engine-stretch|cfile=versions-py2|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp '
scripts / versions_manager . py freeze - v ' dryrun|cmod=default|cfile=versions-docker|cname=all|stage=sub|stage=add|stage=init|stage=tmodname|stage=tmp '
"""