From 6556c400402fe9e625014386ca4650576f369257 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 3 Aug 2020 11:50:06 -0700 Subject: [PATCH] [201911] Introduce sonic-py-common package (#5063) Consolidate common SONiC Python-language functionality into one shared package (sonic-py-common) and eliminate duplicate code. The package currently includes four modules: - daemon_base - device_info - logger - task_base NOTE: This is a combination of all changes from https://github.com/Azure/sonic-buildimage/pull/5003, https://github.com/Azure/sonic-buildimage/pull/5049 and some changes from https://github.com/Azure/sonic-buildimage/pull/5043 backported to align with the 201911 branch. As part of the 201911 port, I am not installing the Python 3 package in the base image or in the VS container, because we do not have pip3 installed, and we do not intend to migrate to Python 3 in 201911. --- .gitignore | 6 + .../build_templates/sonic_debian_extension.j2 | 18 +- platform/mellanox/mlnx-platform-api.mk | 2 +- platform/vs/docker-sonic-vs.mk | 3 + rules/docker-config-engine-stretch.mk | 1 + rules/docker-config-engine.mk | 1 + rules/docker-platform-monitor.mk | 1 + rules/docker-snmp-sv2.mk | 2 +- rules/sonic-py-common.mk | 15 + rules/sonic-thermalctld.mk | 2 +- slave.mk | 4 + src/sonic-py-common/setup.py | 39 ++ .../sonic_py_common/__init__.py | 0 .../sonic_py_common/daemon_base.py | 81 +++++ .../sonic_py_common/device_info.py | 334 ++++++++++++++++++ src/sonic-py-common/sonic_py_common/logger.py | 106 ++++++ .../sonic_py_common/task_base.py | 50 +++ 17 files changed, 656 insertions(+), 9 deletions(-) create mode 100644 rules/sonic-py-common.mk create mode 100644 src/sonic-py-common/setup.py create mode 100644 src/sonic-py-common/sonic_py_common/__init__.py create mode 100644 src/sonic-py-common/sonic_py_common/daemon_base.py create mode 100644 src/sonic-py-common/sonic_py_common/device_info.py create mode 100644 src/sonic-py-common/sonic_py_common/logger.py create mode 100644 src/sonic-py-common/sonic_py_common/task_base.py diff --git a/.gitignore b/.gitignore index e763b1d26a..faf057bff4 100644 --- a/.gitignore +++ b/.gitignore @@ -141,10 +141,16 @@ installer/x86_64/platforms/ src/sonic-config-engine/**/*.pyc src/sonic-config-engine/build src/sonic-config-engine/sonic_config_engine.egg-info + src/sonic-daemon-base/**/*.pyc src/sonic-daemon-base/build src/sonic-daemon-base/sonic_daemon_base.egg-info +src/sonic-py-common/**/*.pyc +src/sonic-py-common/build +src/sonic-py-common/dist +src/sonic-py-common/sonic_py_common.egg-info + # Misc. files files/initramfs-tools/arista-convertfs files/initramfs-tools/union-mount diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index f84f133dc0..003cc696d9 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -97,12 +97,6 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in python-yaml \ python-bitarray -# Install SONiC config engine Python package -CONFIG_ENGINE_WHEEL_NAME=$(basename {{config_engine_wheel_path}}) -sudo cp {{config_engine_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHEEL_NAME -sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME - # Install Python client for Redis sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6" @@ -121,6 +115,18 @@ sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME +# Install sonic-py-common Python 2 package +SONIC_PY_COMMON_PY2_WHEEL_NAME=$(basename {{sonic_py_common_py2_wheel_path}}) +sudo cp {{sonic_py_common_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY2_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME + +# Install SONiC config engine Python package +CONFIG_ENGINE_WHEEL_NAME=$(basename {{config_engine_wheel_path}}) +sudo cp {{config_engine_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME + # Install sonic-platform-common Python 2 package PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}}) sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME diff --git a/platform/mellanox/mlnx-platform-api.mk b/platform/mellanox/mlnx-platform-api.mk index 7bbbc3c70b..c579b45392 100644 --- a/platform/mellanox/mlnx-platform-api.mk +++ b/platform/mellanox/mlnx-platform-api.mk @@ -3,7 +3,7 @@ SONIC_PLATFORM_API_PY2 = mlnx_platform_api-1.0-py2-none-any.whl $(SONIC_PLATFORM_API_PY2)_SRC_PATH = $(PLATFORM_PATH)/mlnx-platform-api $(SONIC_PLATFORM_API_PY2)_PYTHON_VERSION = 2 -$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE) +$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE) SONIC_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2) export mlnx_platform_api_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2))" diff --git a/platform/vs/docker-sonic-vs.mk b/platform/vs/docker-sonic-vs.mk index 7185342316..41d8e0f121 100644 --- a/platform/vs/docker-sonic-vs.mk +++ b/platform/vs/docker-sonic-vs.mk @@ -13,6 +13,9 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \ $(DOCKER_SONIC_VS)_PYTHON_DEBS += $(SONIC_UTILS) +$(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \ + $(SONIC_PY_COMMON_PY2) + ifeq ($(INSTALL_DEBUG_TOOLS), y) $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \ $(LIBSWSSCOMMON_DBG) \ diff --git a/rules/docker-config-engine-stretch.mk b/rules/docker-config-engine-stretch.mk index 2a1e36d288..497593a316 100644 --- a/rules/docker-config-engine-stretch.mk +++ b/rules/docker-config-engine-stretch.mk @@ -3,6 +3,7 @@ DOCKER_CONFIG_ENGINE_STRETCH = docker-config-engine-stretch.gz $(DOCKER_CONFIG_ENGINE_STRETCH)_PATH = $(DOCKERS_PATH)/docker-config-engine-stretch $(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SWSSSDK_PY2) +$(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE) $(DOCKER_CONFIG_ENGINE_STRETCH)_LOAD_DOCKERS += $(DOCKER_BASE_STRETCH) diff --git a/rules/docker-config-engine.mk b/rules/docker-config-engine.mk index f540bb66f3..5d65314b67 100644 --- a/rules/docker-config-engine.mk +++ b/rules/docker-config-engine.mk @@ -3,6 +3,7 @@ DOCKER_CONFIG_ENGINE = docker-config-engine.gz $(DOCKER_CONFIG_ENGINE)_PATH = $(DOCKERS_PATH)/docker-config-engine $(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SWSSSDK_PY2) +$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE) $(DOCKER_CONFIG_ENGINE)_LOAD_DOCKERS += $(DOCKER_BASE) SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE) diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk index db1c8c5a02..2897e90678 100644 --- a/rules/docker-platform-monitor.mk +++ b/rules/docker-platform-monitor.mk @@ -13,6 +13,7 @@ endif $(DOCKER_PLATFORM_MONITOR)_PYTHON_DEBS += $(SONIC_LEDD) $(SONIC_XCVRD) $(SONIC_PSUD) $(SONIC_SYSEEPROMD) $(SONIC_THERMALCTLD) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2) +$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_DAEMON_BASE_PY2) diff --git a/rules/docker-snmp-sv2.mk b/rules/docker-snmp-sv2.mk index 59f99ac78b..91cfa0f6c0 100644 --- a/rules/docker-snmp-sv2.mk +++ b/rules/docker-snmp-sv2.mk @@ -14,7 +14,7 @@ $(DOCKER_SNMP_SV2)_DBG_DEPENDS += $(SNMP_DBG) $(SNMPD_DBG) $(LIBSNMP_DBG) $(DOCKER_SNMP_SV2)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) -$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3) +$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3) $(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2) diff --git a/rules/sonic-py-common.mk b/rules/sonic-py-common.mk new file mode 100644 index 0000000000..b730ef5585 --- /dev/null +++ b/rules/sonic-py-common.mk @@ -0,0 +1,15 @@ +# SONIC_PY_COMMON_PY2 package + +SONIC_PY_COMMON_PY2 = sonic_py_common-1.0-py2-none-any.whl +$(SONIC_PY_COMMON_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-common +$(SONIC_PY_COMMON_PY2)_DEPENDS += $(SWSSSDK_PY2) +$(SONIC_PY_COMMON_PY2)_PYTHON_VERSION = 2 +SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) + +# SONIC_PY_COMMON_PY3 package + +SONIC_PY_COMMON_PY3 = sonic_py_common-1.0-py3-none-any.whl +$(SONIC_PY_COMMON_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-common +$(SONIC_PY_COMMON_PY3)_DEPENDS += $(SWSSSDK_PY3) +$(SONIC_PY_COMMON_PY3)_PYTHON_VERSION = 3 +SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) diff --git a/rules/sonic-thermalctld.mk b/rules/sonic-thermalctld.mk index 775082e7bb..cf42c2544a 100644 --- a/rules/sonic-thermalctld.mk +++ b/rules/sonic-thermalctld.mk @@ -2,5 +2,5 @@ SONIC_THERMALCTLD = python-sonic-thermalctld_1.0-1_all.deb $(SONIC_THERMALCTLD)_SRC_PATH = $(SRC_PATH)/sonic-platform-daemons/sonic-thermalctld -$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_DAEMON_BASE_PY2) +$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) SONIC_PYTHON_STDEB_DEBS += $(SONIC_THERMALCTLD) diff --git a/slave.mk b/slave.mk index 0c4f2c70b3..f066030d56 100644 --- a/slave.mk +++ b/slave.mk @@ -636,6 +636,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(DEBS_PATH)/,$(SONIC_ZTP))) \ $(addprefix $(STRETCH_FILES_PATH)/, $(if $(filter $(CONFIGURED_ARCH),amd64), $(IXGBE_DRIVER))) \ $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \ @@ -664,6 +666,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export installer_debs="$(addprefix $(STRETCH_DEBS_PATH)/,$($*_INSTALLS))" export lazy_installer_debs="$(foreach deb, $($*_LAZY_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(STRETCH_DEBS_PATH)/$(deb))))" export installer_images="$(addprefix $(TARGET_PATH)/,$($*_DOCKERS))" + export sonic_py_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2))" + export sonic_py_common_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3))" export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))" export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))" export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))" diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py new file mode 100644 index 0000000000..b129447262 --- /dev/null +++ b/src/sonic-py-common/setup.py @@ -0,0 +1,39 @@ +from setuptools import setup + +dependencies = [ + 'natsort', + 'pyyaml', + 'swsssdk>=2.0.1', +] + +high_performance_deps = [ + 'swsssdk[high_perf]>=2.0.1', +] + +setup( + name='sonic-py-common', + version='1.0', + description='Common Python libraries for SONiC', + license='Apache 2.0', + author='SONiC Team', + author_email='linuxnetdev@microsoft.com', + url='https://github.com/Azure/SONiC', + maintainer='Joe LeVeque', + maintainer_email='jolevequ@microsoft.com', + install_requires=dependencies, + extras_require={ + 'high_perf': high_performance_deps, + }, + packages=[ + 'sonic_py_common', + ], + classifiers=[ + 'Intended Audience :: Developers', + 'Operating System :: Linux', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python', + ], + keywords='SONiC sonic PYTHON python COMMON common', +) + diff --git a/src/sonic-py-common/sonic_py_common/__init__.py b/src/sonic-py-common/sonic_py_common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py new file mode 100644 index 0000000000..77188a004f --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -0,0 +1,81 @@ +import imp +import signal +import sys + +from . import device_info +from .logger import Logger + +# +# Constants ==================================================================== +# +REDIS_TIMEOUT_MSECS = 0 + +EEPROM_MODULE_NAME = 'eeprom' +EEPROM_CLASS_NAME = 'board' + +# +# Helper functions ============================================================= +# + +def db_connect(db_name): + from swsscommon import swsscommon + return swsscommon.DBConnector(db_name, REDIS_TIMEOUT_MSECS, True) + +# +# DaemonBase =================================================================== +# + +class DaemonBase(Logger): + def __init__(self, log_identifier): + super(DaemonBase, self).__init__(log_identifier, Logger.LOG_FACILITY_DAEMON) + + # Register our default signal handlers, unless the signal already has a + # handler registered, most likely from a subclass implementation + if not signal.getsignal(signal.SIGHUP): + signal.signal(signal.SIGHUP, self.signal_handler) + if not signal.getsignal(signal.SIGINT): + signal.signal(signal.SIGINT, self.signal_handler) + if not signal.getsignal(signal.SIGTERM): + signal.signal(signal.SIGTERM, self.signal_handler) + + # Default signal handler; can be overridden by subclass + def signal_handler(self, sig, frame): + if sig == signal.SIGHUP: + log_info("DaemonBase: Caught SIGHUP - ignoring...") + elif sig == signal.SIGINT: + log_info("DaemonBase: Caught SIGINT - exiting...") + sys.exit(128 + sig) + elif sig == signal.SIGTERM: + log_info("DaemonBase: Caught SIGTERM - exiting...") + sys.exit(128 + sig) + else: + log_warning("DaemonBase: Caught unhandled signal '{}'".format(sig)) + + # Loads platform specific platform module from source + def load_platform_util(self, module_name, class_name): + platform_util = None + + # Get path to platform and hwsku + (platform_path, hwsku_path) = device_info.get_paths_to_platform_and_hwsku() + + try: + module_file = "/".join([platform_path, "plugins", module_name + ".py"]) + module = imp.load_source(module_name, module_file) + except IOError as e: + raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e))) + + try: + platform_util_class = getattr(module, class_name) + # board class of eeprom requires 4 paramerters, need special treatment here. + if module_name == EEPROM_MODULE_NAME and class_name == EEPROM_CLASS_NAME: + platform_util = platform_util_class('','','','') + else: + platform_util = platform_util_class() + except AttributeError as e: + raise AttributeError("Failed to instantiate '%s' class: %s" % (class_name, str(e))) + + return platform_util + + # Runs daemon + def run(self): + raise NotImplementedError() diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py new file mode 100644 index 0000000000..a1645182a8 --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -0,0 +1,334 @@ +import glob +import os +import re +import subprocess + +import yaml +from natsort import natsorted + +# TODD: Replace with swsscommon +from swsssdk import ConfigDBConnector, SonicDBConfig + +USR_SHARE_SONIC_PATH = "/usr/share/sonic" +HOST_DEVICE_PATH = USR_SHARE_SONIC_PATH + "/device" +CONTAINER_PLATFORM_PATH = USR_SHARE_SONIC_PATH + "/platform" + +MACHINE_CONF_PATH = "/host/machine.conf" +SONIC_VERSION_YAML_PATH = "/etc/sonic/sonic_version.yml" + +# Port configuration file names +PORT_CONFIG_FILE = "port_config.ini" +PLATFORM_JSON_FILE = "platform.json" + +# Multi-NPU constants +# TODO: Move Multi-ASIC-related functions and constants to a "multi_asic.py" module +NPU_NAME_PREFIX = "asic" +NAMESPACE_PATH_GLOB = "/run/netns/*" +ASIC_CONF_FILENAME = "asic.conf" +FRONTEND_ASIC_SUB_ROLE = "FrontEnd" +BACKEND_ASIC_SUB_ROLE = "BackEnd" + + +def get_machine_info(): + """ + Retreives data from the machine configuration file + + Returns: + A dictionary containing the key/value pairs as found in the machine + configuration file + """ + if not os.path.isfile(MACHINE_CONF_PATH): + return None + + machine_vars = {} + with open(MACHINE_CONF_PATH) as machine_conf_file: + for line in machine_conf_file: + tokens = line.split('=') + if len(tokens) < 2: + continue + machine_vars[tokens[0]] = tokens[1].strip() + + return machine_vars + + +def get_platform(): + """ + Retrieve the device's platform identifier + + Returns: + A string containing the device's platform identifier + """ + machine_info = get_machine_info() + if machine_info: + if 'onie_platform' in machine_info: + return machine_info['onie_platform'] + elif 'aboot_platform' in machine_info: + return machine_info['aboot_platform'] + + return None + + +def get_hwsku(): + """ + Retrieve the device's hardware SKU identifier + + Returns: + A string containing the device's hardware SKU identifier + """ + config_db = ConfigDBConnector() + config_db.connect() + + metadata = config_db.get_table('DEVICE_METADATA') + + if 'localhost' in metadata and 'hwsku' in metadata['localhost']: + return metadata['localhost']['hwsku'] + + return "" + + +def get_platform_and_hwsku(): + """ + Convenience function which retrieves both the device's platform identifier + and hardware SKU identifier + + Returns: + A tuple of two strings, the first containing the device's + platform identifier, the second containing the device's + hardware SKU identifier + """ + platform = get_platform() + hwsku = get_hwsku() + + return (platform, hwsku) + + +def get_asic_conf_file_path(): + """ + Retrieves the path to the ASIC conguration file on the device + + Returns: + A string containing the path to the ASIC conguration file on success, + None on failure + """ + asic_conf_path_candidates = [] + + asic_conf_path_candidates.append(os.path.join(CONTAINER_PLATFORM_PATH, ASIC_CONF_FILENAME)) + + platform = get_platform() + if platform: + asic_conf_path_candidates.append(os.path.join(HOST_DEVICE_PATH, platform, ASIC_CONF_FILENAME)) + + for asic_conf_file_path in asic_conf_path_candidates: + if os.path.isfile(asic_conf_file_path): + return asic_conf_file_path + + return None + + +def get_paths_to_platform_and_hwsku_dirs(): + """ + Retreives the paths to the device's platform and hardware SKU data + directories + + Returns: + A tuple of two strings, the first containing the path to the platform + directory of the device, the second containing the path to the hardware + SKU directory of the device + """ + # Get platform and hwsku + (platform, hwsku) = get_platform_and_hwsku() + + # Determine whether we're running in a container or on the host + platform_path_host = os.path.join(HOST_DEVICE_PATH, platform) + + if os.path.isdir(CONTAINER_PLATFORM_PATH): + platform_path = CONTAINER_PLATFORM_PATH + elif os.path.isdir(platform_path_host): + platform_path = platform_path_host + else: + raise OSError("Failed to locate platform directory") + + hwsku_path = os.path.join(platform_path, hwsku) + + return (platform_path, hwsku_path) + + +def get_path_to_port_config_file(): + """ + Retrieves the path to the device's port configuration file + + Returns: + A string containing the path the the device's port configuration file + """ + # Get platform and hwsku path + (platform_path, hwsku_path) = get_paths_to_platform_and_hwsku_dirs() + + # First check for the presence of the new 'platform.json' file + port_config_file_path = os.path.join(platform_path, PLATFORM_JSON_FILE) + if not os.path.isfile(port_config_file_path): + # platform.json doesn't exist. Try loading the legacy 'port_config.ini' file + port_config_file_path = os.path.join(hwsku_path, PORT_CONFIG_FILE) + if not os.path.isfile(port_config_file_path): + raise OSError("Failed to detect port config file: {}".format(port_config_file_path)) + + return port_config_file_path + + +def get_sonic_version_info(): + if not os.path.isfile(SONIC_VERSION_YAML_PATH): + return None + + data = {} + with open(SONIC_VERSION_YAML_PATH) as stream: + if yaml.__version__ >= "5.1": + data = yaml.full_load(stream) + else: + data = yaml.load(stream) + + return data + + +# +# Multi-NPU functionality +# + +def get_num_npus(): + asic_conf_file_path = get_asic_conf_file_path() + if asic_conf_file_path is None: + return 1 + with open(asic_conf_file_path) as asic_conf_file: + for line in asic_conf_file: + tokens = line.split('=') + if len(tokens) < 2: + continue + if tokens[0].lower() == 'num_asic': + num_npus = tokens[1].strip() + return int(num_npus) + + +def is_multi_npu(): + num_npus = get_num_npus() + return (num_npus > 1) + + +def get_npu_id_from_name(npu_name): + if npu_name.startswith(NPU_NAME_PREFIX): + return npu_name[len(NPU_NAME_PREFIX):] + else: + return None + + +def get_namespaces(): + """ + In a multi NPU platform, each NPU is in a Linux Namespace. + This method returns list of all the Namespace present on the device + """ + ns_list = [] + for path in glob.glob(NAMESPACE_PATH_GLOB): + ns = os.path.basename(path) + ns_list.append(ns) + return natsorted(ns_list) + + +def get_all_namespaces(): + """ + In case of Multi-Asic platform, Each ASIC will have a linux network namespace created. + So we loop through the databases in different namespaces and depending on the sub_role + decide whether this is a front end ASIC/namespace or a back end one. + """ + front_ns = [] + back_ns = [] + num_npus = get_num_npus() + SonicDBConfig.load_sonic_global_db_config() + + if is_multi_npu(): + for npu in range(num_npus): + namespace = "{}{}".format(NPU_NAME_PREFIX, npu) + config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace) + config_db.connect() + + metadata = config_db.get_table('DEVICE_METADATA') + if metadata['localhost']['sub_role'] == FRONTEND_ASIC_SUB_ROLE: + front_ns.append(namespace) + elif metadata['localhost']['sub_role'] == BACKEND_ASIC_SUB_ROLE: + back_ns.append(namespace) + + return {'front_ns':front_ns, 'back_ns':back_ns} + + +def _valid_mac_address(mac): + return bool(re.match("^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$", mac)) + + +def get_system_mac(namespace=None): + version_info = get_sonic_version_info() + + if (version_info['asic_type'] == 'mellanox'): + # With Mellanox ONIE release(2019.05-5.2.0012) and above + # "onie_base_mac" was added to /host/machine.conf: + # onie_base_mac=e4:1d:2d:44:5e:80 + # So we have another way to get the mac address besides decode syseeprom + # By this can mitigate the dependency on the hw-management service + base_mac_key = "onie_base_mac" + machine_vars = get_machine_info() + if machine_vars is not None and base_mac_key in machine_vars: + mac = machine_vars[base_mac_key] + mac = mac.strip() + if _valid_mac_address(mac): + return mac + + hw_mac_entry_cmds = [ "sudo decode-syseeprom -m" ] + elif (version_info['asic_type'] == 'marvell'): + # Try valid mac in eeprom, else fetch it from eth0 + platform = get_platform() + hwsku = get_hwsku() + profile_cmd = 'cat' + HOST_DEVICE_PATH + '/' + platform +'/'+ hwsku +'/profile.ini | grep switchMacAddress | cut -f2 -d=' + hw_mac_entry_cmds = [ profile_cmd, "sudo decode-syseeprom -m", "ip link show eth0 | grep ether | awk '{print $2}'" ] + else: + mac_address_cmd = "cat /sys/class/net/eth0/address" + if namespace is not None: + mac_address_cmd = "sudo ip netns exec {} {}".format(namespace, mac_address_cmd) + + hw_mac_entry_cmds = [mac_address_cmd] + + for get_mac_cmd in hw_mac_entry_cmds: + proc = subprocess.Popen(get_mac_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (mac, err) = proc.communicate() + if err: + continue + mac = mac.strip() + if _valid_mac_address(mac): + break + + if not _valid_mac_address(mac): + return None + + # Align last byte of MAC if necessary + if version_info and version_info['asic_type'] == 'centec': + last_byte = mac[-2:] + aligned_last_byte = format(int(int(last_byte, 16) + 1), '02x') + mac = mac[:-2] + aligned_last_byte + return mac + + +def get_system_routing_stack(): + """ + Retrieves the routing stack being utilized on this device + + Returns: + A string containing the name of the routing stack in use on the device + """ + command = "sudo docker ps | grep bgp | awk '{print$2}' | cut -d'-' -f3 | cut -d':' -f1" + + try: + proc = subprocess.Popen(command, + stdout=subprocess.PIPE, + shell=True, + stderr=subprocess.STDOUT) + stdout = proc.communicate()[0] + proc.wait() + result = stdout.rstrip('\n') + except OSError as e: + raise OSError("Cannot detect routing stack") + + return result diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py new file mode 100644 index 0000000000..26adf999a0 --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -0,0 +1,106 @@ +import os +import sys +import syslog + +""" +Logging functionality for SONiC Python applications +""" + + +class Logger(object): + """ + Logger class for SONiC Python applications + """ + LOG_FACILITY_USER = syslog.LOG_USER + LOG_FACILITY_DAEMON = syslog.LOG_DAEMON + + LOG_PRIORITY_ERROR = syslog.LOG_ERR + LOG_PRIORITY_WARNING = syslog.LOG_WARNING + LOG_PRIORITY_NOTICE = syslog.LOG_NOTICE + LOG_PRIORITY_INFO = syslog.LOG_INFO + LOG_PRIORITY_DEBUG = syslog.LOG_DEBUG + + def __init__(self, log_identifier=None, log_facility=LOG_FACILITY_USER): + self.syslog = syslog + + if not log_identifier: + log_identifier = os.path.basename(sys.argv[0]) + + self.syslog.openlog(ident=log_identifier, + logoption=(syslog.LOG_PID | syslog.LOG_NDELAY), + facility=log_facility) + + # Set the default minimum log priority to LOG_PRIORITY_NOTICE + self.set_min_log_priority(self.LOG_PRIORITY_NOTICE) + + def __del__(self): + self.syslog.closelog() + + # + # Methods for setting minimum log priority + # + + def set_min_log_priority(self, priority): + """ + Sets the minimum log priority level to . All log messages + with a priority lower than will not be logged + + Args: + priority: The minimum priority at which to log messages + """ + self.syslog.setlogmask(self.syslog.LOG_UPTO(priority)) + + def set_min_log_priority_error(self): + """ + Convenience function to set minimum log priority to LOG_PRIORITY_ERROR + """ + self.set_min_log_priority(self.LOG_PRIORITY_ERROR) + + def set_min_log_priority_warning(self): + """ + Convenience function to set minimum log priority to LOG_PRIORITY_WARNING + """ + self.set_min_log_priority(self.LOG_PRIORITY_WARNING) + + def set_min_log_priority_notice(self): + """ + Convenience function to set minimum log priority to LOG_PRIORITY_NOTICE + """ + self.set_min_log_priority(self.LOG_PRIORITY_NOTICE) + + def set_min_log_priority_info(self): + """ + Convenience function to set minimum log priority to LOG_PRIORITY_INFO + """ + self.set_min_log_priority(self.LOG_PRIORITY_INFO) + + def set_min_log_priority_debug(self): + """ + Convenience function to set minimum log priority to LOG_PRIORITY_DEBUG + """ + self.set_min_log_priority(self.LOG_PRIORITY_DEBUG) + + # + # Methods for logging messages + # + + def log(self, priority, msg, also_print_to_console=False): + self.syslog.syslog(priority, msg) + + if also_print_to_console: + print(msg) + + def log_error(self, msg, also_print_to_console=False): + self.log(self.LOG_PRIORITY_ERROR, msg, also_print_to_console) + + def log_warning(self, msg, also_print_to_console=False): + self.log(self.LOG_PRIORITY_WARNING, msg, also_print_to_console) + + def log_notice(self, msg, also_print_to_console=False): + self.log(self.LOG_PRIORITY_NOTICE, msg, also_print_to_console) + + def log_info(self, msg, also_print_to_console=False): + self.log(self.LOG_PRIORITY_INFO, msg, also_print_to_console) + + def log_debug(self, msg, also_print_to_console=False): + self.log(self.LOG_PRIORITY_DEBUG, msg, also_print_to_console) diff --git a/src/sonic-py-common/sonic_py_common/task_base.py b/src/sonic-py-common/sonic_py_common/task_base.py new file mode 100644 index 0000000000..e1738ffba2 --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/task_base.py @@ -0,0 +1,50 @@ +import multiprocessing +import os +import signal +import threading + + +# +# ProcessTaskBase ===================================================================== +# +class ProcessTaskBase(object): # TODO: put this class to swss-platform-common + def __init__(self): + self.task_process = None + self.task_stopping_event = multiprocessing.Event() + + def task_worker(self): + pass + + def task_run(self): + if self.task_stopping_event.is_set(): + return + + self.task_process = multiprocessing.Process(target=self.task_worker) + self.task_process.start() + + def task_stop(self): + self.task_stopping_event.set() + os.kill(self.task_process.pid, signal.SIGKILL) + + +# +# ThreadTaskBase ===================================================================== +# +class ThreadTaskBase(object): # TODO: put this class to swss-platform-common; + def __init__(self): + self.task_thread = None + self.task_stopping_event = threading.Event() + + def task_worker(self): + pass + + def task_run(self): + if self.task_stopping_event.is_set(): + return + + self.task_thread = threading.Thread(target=self.task_worker) + self.task_thread.start() + + def task_stop(self): + self.task_stopping_event.set() + self.task_thread.join()