matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add sonic-host-services and sonic-host-services-data packages (#5694)

Browse Source 
**- Why I did it**

Install all host services and their data files in package format rather than file-by-file

**- How I did it**

- Create sonic-host-services Python wheel package, currently including procdockerstatsd
  - Also add the framework for unit tests by adding one simple procdockerstatsd test case
- Create sonic-host-services-data Debian package which is responsible for installing the related systemd unit files to control the services in the Python wheel. This package will also be responsible for installing any Jinja2 templates and other data files needed by the host services.
This commit is contained in:
Joe LeVeque 2020-10-23 09:52:29 -07:00 committed by GitHub
parent bb641913c4
commit 3a4435eb53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 247 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
files/build_templates/sonic_debian_extension.j2
View File

@ -206,6 +206,16 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
# in bash.bashrc, so we copy a version of the file with it enabled here.
sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
# Install SONiC host services package
SONIC_HOST_SERVICES_PY3_WHEEL_NAME=$(basename {{sonic_host_services_py3_wheel_path}})
sudo cp {{sonic_host_services_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_HOST_SERVICES_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
# Install SONiC host services data files (and any dependencies via 'apt-get -y install -f')
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-services-data_*.deb || \
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
{% if enable_ztp == "y" %}
# Install ZTP (and its dependencies via 'apt-get -y install -f')
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
@ -402,11 +412,6 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEM
echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE
sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/
# Copy process/docker cpu/memory utilization data export daemon
sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE
sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd $FILESYSTEM_ROOT/usr/bin/
# Copy systemd timer configuration
sudo cp $BUILD_TEMPLATES/pcie-check.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable pcie-check.timer

6
platform/vs/docker-sonic-vs.mk
View File

@ -12,7 +12,8 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \
$(LIBYANG) \
$(LIBYANG_CPP) \
$(LIBYANG_PY2) \
$(SONIC_UTILITIES_DATA)
$(SONIC_UTILITIES_DATA) \
$(SONIC_HOST_SERVICES_DATA)
# swsssdk is a dependency of sonic-py-common
# TODO: sonic-py-common should depend on swsscommon instead
@ -22,7 +23,8 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
$(SONIC_PY_COMMON_PY3) \
$(SONIC_YANG_MODELS_PY3) \
$(SONIC_YANG_MGMT_PY) \
$(SONIC_UTILITIES_PY2)
$(SONIC_UTILITIES_PY2) \
$(SONIC_HOST_SERVICES_PY3)
ifeq ($(INSTALL_DEBUG_TOOLS), y)
$(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \

8
rules/sonic-host-services-data.dep Normal file
View File

@ -0,0 +1,8 @@
SPATH := $($(SONIC_HOST_SERVICES_DATA)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services-data.mk rules/sonic-host-services-data.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(SPATH))
$(SONIC_HOST_SERVICES_DATA)_CACHE_MODE := GIT_CONTENT_SHA
$(SONIC_HOST_SERVICES_DATA)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(SONIC_HOST_SERVICES_DATA)_DEP_FILES := $(DEP_FILES)

5
rules/sonic-host-services-data.mk Normal file
View File

@ -0,0 +1,5 @@
# SONiC host services data package
SONIC_HOST_SERVICES_DATA = sonic-host-services-data_1.0-1_all.deb
$(SONIC_HOST_SERVICES_DATA)_SRC_PATH = $(SRC_PATH)/sonic-host-services-data
SONIC_DPKG_DEBS += $(SONIC_HOST_SERVICES_DATA)

10
rules/sonic-host-services.dep Normal file
View File

@ -0,0 +1,10 @@
SPATH := $($(SONIC_HOST_SERVICES_PY3)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services.mk rules/sonic-host-services.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
$(SONIC_HOST_SERVICES_PY3)_CACHE_MODE := GIT_CONTENT_SHA
$(SONIC_HOST_SERVICES_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(SONIC_HOST_SERVICES_PY3)_DEP_FILES := $(DEP_FILES)
$(SONIC_HOST_SERVICES_PY3)_SMDEP_FILES := $(SMDEP_FILES)
$(SONIC_HOST_SERVICES_PY3)_SMDEP_PATHS := $(SPATH)

8
rules/sonic-host-services.mk Normal file
View File

@ -0,0 +1,8 @@
# SONiC host services package
SONIC_HOST_SERVICES_PY3 = sonic_host_services-1.0-py3-none-any.whl
$(SONIC_HOST_SERVICES_PY3)_SRC_PATH = $(SRC_PATH)/sonic-host-services
$(SONIC_HOST_SERVICES_PY3)_PYTHON_VERSION = 3
$(SONIC_HOST_SERVICES_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \
$(SWSSSDK_PY3)
SONIC_PYTHON_WHEELS += $(SONIC_HOST_SERVICES_PY3)

7
slave.mk
View File

@ -804,7 +804,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(MONIT) \
$(PYTHON_SWSSCOMMON) \
$(PYTHON3_SWSSCOMMON) \
$(SONIC_UTILITIES_DATA)) \
$(SONIC_UTILITIES_DATA) \
$(SONIC_HOST_SERVICES_DATA)) \
$$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
$$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
$(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
@ -820,7 +821,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH))
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
$(HEADER)
# Pass initramfs and linux kernel explicitly. They are used for all platforms
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@ -866,6 +868,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"
export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))"
export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))"
$(foreach docker, $($*_DOCKERS),\
export docker_image="$(docker)"

6
src/sonic-host-services-data/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
debian/*.debhelper
debian/debhelper-build-stamp
debian/sonic-host-services-data/
sonic-host-services-data_*.buildinfo
sonic-host-services-data_*.changes
sonic-host-services-data_*.deb

15
src/sonic-host-services-data/LICENSE Normal file
View File

@ -0,0 +1,15 @@
Copyright (C) 2020 Microsoft
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

7
src/sonic-host-services-data/MAINTAINERS Normal file
View File

@ -0,0 +1,7 @@
# This file describes the maintainers for sonic-device-data
# See the SONiC project governance document for more information
Name = "Joe LeVeque"
Email = "jolevequ@microsoft.com"
Github = jleveque
Mailinglist = sonicproject@googlegroups.com

19
src/sonic-host-services-data/README.md Normal file
View File

@ -0,0 +1,19 @@
# sonic-host-services-data
Data files required for SONiC host services
## To build
```
dpkg-buildpackage -rfakeroot -b -us -uc
```
## To clean
```
dpkg-buildpackage -rfakeroot -Tclean
```
---
See the [SONiC Website](http://azure.github.io/SONiC/) for more information about the SONiC project.

5
src/sonic-host-services-data/debian/changelog Normal file
View File

@ -0,0 +1,5 @@
sonic-host-services-data (1.0-1) UNRELEASED; urgency=low
* Initial release
-- Joe LeVeque <jolevequ@microsoft.com> Tue, 20 Oct 2020 02:35:43 +0000

1
src/sonic-host-services-data/debian/compat Normal file
View File

@ -0,0 +1 @@
11

11
src/sonic-host-services-data/debian/control Normal file
View File

@ -0,0 +1,11 @@
Source: sonic-host-services-data
Maintainer: Joe LeVeque <jolevequ@microsoft.com>
Section: misc
Priority: optional
Standards-Version: 0.1
Build-Depends: debhelper (>=11)
Package: sonic-host-services-data
Architecture: all
Depends: ${misc:Depends}
Description: Data files required for SONiC host services

0
src/sonic-host-services-data/debian/copyright Normal file
View File

0
src/sonic-host-services-data/debian/install Normal file
View File

9
src/sonic-host-services-data/debian/rules Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/make -f
build:
%:
dh $@
override_dh_installsystemd:
dh_installsystemd --no-start --name=procdockerstatsd

2
files/image_config/procdockerstatsd/procdockerstatsd.service → src/sonic-host-services-data/debian/sonic-host-services-data.procdockerstatsd.service
View File

@ -5,7 +5,7 @@ After=database.service updategraph.service
[Service]
Type=simple
ExecStart=/usr/bin/procdockerstatsd
ExecStart=/usr/local/bin/procdockerstatsd
Restart=always
[Install]

14
src/sonic-host-services/.gitignore vendored Normal file
View File

@ -0,0 +1,14 @@
# Compiled Python files
*.pyc
scripts/procdockerstatsdc
# Generated by packaging
*.egg-info/
.eggs/
build/
dist/
# Unit test coverage
.coverage
coverage.xml
htmlcov/

0
files/image_config/procdockerstatsd/procdockerstatsd → src/sonic-host-services/scripts/procdockerstatsd
View File

2
src/sonic-host-services/setup.cfg Normal file
View File

@ -0,0 +1,2 @@
[aliases]
test=pytest

41
src/sonic-host-services/setup.py Normal file
View File

@ -0,0 +1,41 @@
from setuptools import setup
setup(
name = 'sonic-host-services',
version = '1.0',
description = 'Python services which run in the SONiC host OS',
license = 'Apache 2.0',
author = 'SONiC Team',
author_email = 'linuxnetdev@microsoft.com',
url = 'https://github.com/Azure/sonic-buildimage',
maintainer = 'Joe LeVeque',
maintainer_email = 'jolevequ@microsoft.com',
scripts = [
'scripts/procdockerstatsd',
],
install_requires = [
'sonic-py-common',
'swsssdk>=2.0.1',
],
setup_requires = [
'pytest-runner',
'wheel'
],
tests_require = [
'pytest',
],
classifiers = [
'Development Status :: 3 - Alpha',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Information Technology',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: Apache Software License',
'Natural Language :: English',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 3.7',
'Topic :: System',
],
keywords = 'sonic SONiC host services',
test_suite = 'setup.get_test_suite'
)

0
src/sonic-host-services/tests/__init__.py Normal file
View File

24
src/sonic-host-services/tests/mock_connector.py Normal file
View File

@ -0,0 +1,24 @@
class MockConnector(object):
STATE_DB = None
data = {}
def __init__(self, host):
pass
def connect(self, db_id):
pass
def get(self, db_id, key, field):
return MockConnector.data[key][field]
def keys(self, db_id, pattern):
match = pattern.split('*')[0]
ret = []
for key in MockConnector.data.keys():
if match in key:
ret.append(key)
return ret
def get_all(self, db_id, key):
return MockConnector.data[key]

42
src/sonic-host-services/tests/procdockerstatsd_test.py Normal file
View File

@ -0,0 +1,42 @@
import imp
import sys
import os
import pytest
import swsssdk
from .mock_connector import MockConnector
swsssdk.SonicV2Connector = MockConnector
test_path = os.path.dirname(os.path.abspath(__file__))
modules_path = os.path.dirname(test_path)
scripts_path = os.path.join(modules_path, "scripts")
sys.path.insert(0, modules_path)
imp.load_source('procdockerstatsd', scripts_path + '/procdockerstatsd')
from procdockerstatsd import *
class TestProcDockerStatsDaemon(object):
def test_convert_to_bytes(self):
test_data = [
('1B', 1),
('500B', 500),
('1KB', 1000),
('500KB', 500000),
('1MB', 1000000),
('500MB', 500000000),
('1MiB', 1048576),
('500MiB', 524288000),
('66.41MiB', 69635932),
('333.6MiB', 349804954),
('1GiB', 1073741824),
('500GiB', 536870912000),
('7.751GiB', 8322572878)
]
pdstatsd = ProcDockerStats(SYSLOG_IDENTIFIER)
for test_input, expected_output in test_data:
res = pdstatsd.convert_to_bytes(test_input)
assert res == expected_output