[sonic-host-service] Move to sonic-host-services package (#6273)

- Why I did it

To move ‘sonic-host-service’ which is currently built as a separate package to ‘sonic-host-services' package. 

- How I did it

- Moved 'sonic-host-server' to 'src/sonic-host-services' and included it as part of the python3 wheel.
- Other files were moved to 'src/sonic-host-services-data' and included as part of the deb package.
- Changed build option ‘INCLUDE_HOST_SERVICE’ to ‘ENABLE_HOST_SERVICE_ON_START’ for enabling sonic-hostservice at boot-up by default.

Makefile.work

@@ -246,6 +246,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 

build_debian.sh

@@ -214,7 +214,9 @@ else
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install docker-ce=${DOCKER_VERSION}
 fi
 
-sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2
+# Uninstall 'python3-gi' installed as part of 'software-properties-common' to remove debian version of 'PyGObject'
+# pip version of 'PyGObject' will be installed during installation of 'sonic-host-services'
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2 python3-gi
 
 if [ "$INCLUDE_KUBERNETES" == "y" ]
 then

files/build_templates/sonic_debian_extension.j2

@@ -246,6 +246,13 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
 
+# Install prerequisites needed for installing the dependent Python packages of sonic-host-services
+# These packages can be uninstalled after installation
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install libcairo2-dev libdbus-1-dev libgirepository1.0-dev libsystemd-dev pkg-config
+
+# Manually install runtime dependencies to avoid them being auto-removed while uninstalling build dependencies
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install gir1.2-glib-2.0 libdbus-1-3 libgirepository-1.0-1 libsystemd0
+
 # Install SONiC host services package
 SONIC_HOST_SERVICES_PY3_WHEEL_NAME=$(basename {{sonic_host_services_py3_wheel_path}})
 sudo cp {{sonic_host_services_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
@@ -262,12 +269,6 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 {% endif %}
 
-{% if include_host_service == "y" %}
-# Install SONiC Host Service (and its dependencies via 'apt-get -y install -f')
-sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-service_*.deb || \
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
-{% endif %}
-
 # Install SONiC Device Data  (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-device-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
@@ -676,6 +677,7 @@ echo "mgmt-framework.timer" | sudo tee -a $GENERATED_SERVICE_FILE
 
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev python3-dev
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y build-essential libssl-dev swig
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y libcairo2-dev libdbus-1-dev libgirepository1.0-dev libsystemd-dev pkg-config
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
 

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -53,7 +53,20 @@ RUN apt-get install -y net-tools \
                        libssl-dev \
                        swig \
                        # For using Python m2crypto package
-                       openssl
+                       openssl \
+                       # For installing dependent Python packages of sonic-host-services
+                       # (these can be uninstalled after installation)
+                       libcairo2-dev \
+                       libdbus-1-dev \
+                       libgirepository1.0-dev \
+                       libsystemd-dev \
+                       pkg-config \
+                       # For installing dependent Python packages of sonic-host-services
+                       # these packages are needed at runtime
+                       gir1.2-glib-2.0 \
+                       libdbus-1-3 \
+                       libgirepository-1.0-1 \
+                       libsystemd0
 
 # Install redis-server
 {% if CONFIGURED_ARCH == "armhf" %}
@@ -129,6 +142,7 @@ RUN pip{% if 'py3' in whl %}3{% else %}2{% endif %} install python-wheels/{{ whl
 # Clean up
 RUN apt-get purge -y build-essential libssl-dev swig
 RUN apt-get purge -y python-dev python3-dev
+RUN apt-get purge -y libcairo2-dev libdbus-1-dev libgirepository1.0-dev libsystemd-dev pkg-config
 RUN apt-get clean -y
 RUN apt-get autoclean -y
 RUN apt-get autoremove -y

rules/config

@@ -129,9 +129,9 @@ INCLUDE_SFLOW = y
 # INCLUDE_MGMT_FRAMEWORK - build docker-sonic-mgmt-framework for CLI and REST server support
 INCLUDE_MGMT_FRAMEWORK = y
 
-# INCLUDE_HOST_SERVICE - build sonic-host-services for mgmt-framework and/or
+# ENABLE_HOST_SERVICE_ON_START - enable sonic-host-server for mgmt-framework and/or
-# telemetry containers to access host functionality
+# telemetry containers to access host functionality by default
-INCLUDE_HOST_SERVICE = n
+ENABLE_HOST_SERVICE_ON_START = n
 
 # INCLUDE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
 INCLUDE_RESTAPI = n

rules/sonic-host-service.dep

@@ -1,12 +0,0 @@
-
-ifeq ($(INCLUDE_HOST_SERVICE), y)
-SPATH       := $($(SONIC_HOST_SERVICE)_SRC_PATH)
-DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-service.mk rules/sonic-host-service.dep
-SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
-
-$(SONIC_HOST_SERVICE)_CACHE_MODE  := GIT_CONTENT_SHA
-$(SONIC_HOST_SERVICE)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
-$(SONIC_HOST_SERVICE)_DEP_FILES   := $(DEP_FILES)
-$(SONIC_HOST_SERVICE)_SMDEP_FILES := $(SMDEP_FILES)
-$(SONIC_HOST_SERVICE)_SMDEP_PATHS := $(SPATH)
-endif

rules/sonic-host-service.mk

@@ -1,9 +0,0 @@
-# SONiC host service package
-
-ifeq ($(INCLUDE_HOST_SERVICE), y)
-
-SONIC_HOST_SERVICE = sonic-host-service_1.0.0_all.deb
-$(SONIC_HOST_SERVICE)_SRC_PATH = $(SRC_PATH)/sonic-host-service
-SONIC_MAKE_DEBS += $(SONIC_HOST_SERVICE)
-
-endif

slave.mk

@@ -247,7 +247,7 @@ $(info "VS_PREPARE_MEM"                  : "$(VS_PREPARE_MEM)")
 $(info "INCLUDE_MGMT_FRAMEWORK"          : "$(INCLUDE_MGMT_FRAMEWORK)")
 $(info "INCLUDE_ICCPD"                   : "$(INCLUDE_ICCPD)")
 $(info "INCLUDE_SYSTEM_TELEMETRY"        : "$(INCLUDE_SYSTEM_TELEMETRY)")
-$(info "INCLUDE_HOST_SERVICE"            : "$(INCLUDE_HOST_SERVICE)")
+$(info "ENABLE_HOST_SERVICE_ON_START"    : "$(ENABLE_HOST_SERVICE_ON_START)")
 $(info "INCLUDE_RESTAPI"                 : "$(INCLUDE_RESTAPI)")
 $(info "INCLUDE_SFLOW"                   : "$(INCLUDE_SFLOW)")
 $(info "INCLUDE_NAT"                     : "$(INCLUDE_NAT)")
@@ -857,7 +857,6 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
         $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
-        $(if $(findstring y,$(INCLUDE_HOST_SERVICE)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_HOST_SERVICE))) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
@@ -892,7 +891,6 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export enable_ztp="$(ENABLE_ZTP)"
 	export include_system_telemetry="$(INCLUDE_SYSTEM_TELEMETRY)"
 	export include_restapi="$(INCLUDE_RESTAPI)"
-	export include_host_service="$(INCLUDE_HOST_SERVICE)"
 	export include_nat="$(INCLUDE_NAT)"
 	export include_sflow="$(INCLUDE_SFLOW)"
 	export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"

sonic-slave-buster/Dockerfile.j2

@@ -339,7 +339,13 @@ RUN apt-get update && apt-get install -y \
         libboost-regex1.71-dev \
         googletest \
         libgtest-dev \
-        libgcc-8-dev
+        libgcc-8-dev \
+# For sonic-host-services build
+        libcairo2-dev \
+        libdbus-1-dev \
+        libgirepository1.0-dev \
+        libsystemd-dev \
+        pkg-config
 
 # Build fix for ARMHF buster libsairedis
 {%- if CONFIGURED_ARCH == "armhf" %}
@@ -390,7 +396,7 @@ RUN export VERSION=1.14.2 \
 
 RUN pip3 install --upgrade pip
 RUN pip2 install --upgrade 'pip<21'
-RUN apt-get purge -y python-pip python3-pip
+RUN apt-get purge -y python-pip python3-pip python3-yaml
 
 # For building Python packages
 RUN pip2 install setuptools==40.8.0

src/sonic-host-service/Makefile

@@ -1,41 +0,0 @@
-################################################################################
-#                                                                              #
-#  Copyright 2020 Dell Inc.                                                    #
-#                                                                              #
-#  Licensed under the Apache License, Version 2.0 (the "License");             #
-#  you may not use this file except in compliance with the License.            #
-#  You may obtain a copy of the License at                                     #
-#                                                                              #
-#     http://www.apache.org/licenses/LICENSE-2.0                               #
-#                                                                              #
-#  Unless required by applicable law or agreed to in writing, software         #
-#  distributed under the License is distributed on an "AS IS" BASIS,           #
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.    #
-#  See the License for the specific language governing permissions and         #
-#  limitations under the License.                                              #
-#                                                                              #
-################################################################################
-
-TOPDIR := $(abspath .)
-MAIN_TARGET = sonic-host-service_1.0.0_all.deb
-INSTALL := /usr/bin/install
-
-$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
-	dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
-	mv ../$(MAIN_TARGET) $(DEST)/
-
-SOURCES := $(wildcard '*.*') $(wildcard 'host_modules/*.py')
-install: $(SOURCES)
-	# Scripts for host service
-	$(INSTALL) -d $(DESTDIR)/usr/lib/sonic_host_service/host_modules
-	$(INSTALL) -D $(TOPDIR)/sonic_host_server.py $(DESTDIR)/usr/lib/sonic_host_service
-	$(INSTALL) -D $(TOPDIR)/host_modules/*.py $(DESTDIR)/usr/lib/sonic_host_service/host_modules
-
-	# D-Bus permissions configuration
-	$(INSTALL) -d $(DESTDIR)/etc/dbus-1/system.d
-	$(INSTALL) -D $(TOPDIR)/org.sonic.hostservice.conf $(DESTDIR)/etc/dbus-1/system.d
-
-	# systemd unit
-	$(INSTALL) -d $(DESTDIR)/lib/systemd/system
-	$(INSTALL) -D $(TOPDIR)/sonic-hostservice.service $(DESTDIR)/lib/systemd/system
-

src/sonic-host-service/debian/.gitignore

@@ -1,4 +0,0 @@
-.debhelper
-sonic-host-service.*
-sonic-host-service/
-files

src/sonic-host-service/debian/changelog

@@ -1,5 +0,0 @@
-sonic-host-service (1.0.0) UNRELEASED; urgency=low
-
-  * Initial release.
-
- -- Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>  Mon, 22 Jun 2020 00:00:00 +0000

src/sonic-host-service/debian/compat

@@ -1 +0,0 @@
-9

src/sonic-host-service/debian/control

@@ -1,15 +0,0 @@
-Source: sonic-host-service
-Maintainer: Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>
-Build-Depends: debhelper (>= 8.0.0),
-               dh-systemd
-Vcs-Git: https://github.com/Azure/sonic-buildimage
-Homepage: https://github.com/Azure/SONiC/
-Standards-Version: 3.9.3
-Section: net
-
-Package: sonic-host-service
-Priority: extra
-Architecture: all
-Depends: python3-systemd, python3-dbus, python3-gi, ${misc:Depends}
-Description: SONiC Host Service
-

src/sonic-host-service/debian/rules

@@ -1,9 +0,0 @@
-#!/usr/bin/make -f
-%:
-	dh $@ --with systemd --parallel
-
-override_dh_auto_clean:
-override_dh_auto_test:
-override_dh_auto_build:
-override_dh_auto_install:
-	make install DESTDIR=debian/sonic-host-service

src/sonic-host-services-data/debian/install

@@ -1 +1,2 @@
 templates/*.j2    /usr/share/sonic/templates/
+org.sonic.hostservice.conf /etc/dbus-1/system.d

src/sonic-host-services-data/debian/rules

@@ -1,5 +1,12 @@
 #!/usr/bin/make -f
 
+ifeq (${ENABLE_HOST_SERVICE_ON_START}, y)
+	HOST_SERVICE_OPTS := --no-start
+else
+	HOST_SERVICE_OPTS := --no-start --no-enable
+endif
+
+
 build:
 
 %:
@@ -11,4 +18,5 @@ override_dh_installsystemd:
 	dh_installsystemd --no-start --name=procdockerstatsd
 	dh_installsystemd --no-start --name=determine-reboot-cause
 	dh_installsystemd --no-start --name=process-reboot-cause
+	dh_installsystemd $(HOST_SERVICE_OPTS) --name=sonic-hostservice
 

src/sonic-host-services-data/debian/sonic-host-services-data.sonic-hostservice.service

@@ -5,7 +5,7 @@ Description=SONiC Host Service
 Type=dbus
 BusName=org.SONiC.HostService
 
-ExecStart=/usr/bin/python3 -u /usr/lib/sonic_host_service/sonic_host_server.py
+ExecStart=/usr/bin/python3 -u /usr/local/bin/sonic-host-server
 
 Restart=on-failure
 RestartSec=10

src/sonic-host-services/scripts/sonic-host-server

@@ -15,7 +15,7 @@ from gi.repository import GObject
 
 def register_modules():
     """Register all host modules"""
-    mod_path = os.path.join(os.path.dirname(__file__), 'host_modules')
+    mod_path = '/usr/local/lib/python3.7/dist-packages/host_modules'
     sys.path.append(mod_path)
     for mod_file in glob.glob(os.path.join(mod_path, '*.py')):
         if os.path.isfile(mod_file) and not mod_file.endswith('__init__.py'):

src/sonic-host-services/setup.py

@@ -10,17 +10,24 @@ setup(
     url = 'https://github.com/Azure/sonic-buildimage',
     maintainer = 'Joe LeVeque',
     maintainer_email = 'jolevequ@microsoft.com',
+    packages = [
+        'host_modules'
+    ],
     scripts = [
         'scripts/caclmgrd',
         'scripts/hostcfgd',
         'scripts/procdockerstatsd',
         'scripts/determine-reboot-cause',
         'scripts/process-reboot-cause',
+        'scripts/sonic-host-server'
     ],
     install_requires = [
+        'dbus-python',
         'Jinja2>=2.10',
+        'PyGObject',
         'sonic-py-common',
         'swsssdk>=2.0.1',
+        'systemd-python',
     ],
     setup_requires = [
         'pytest-runner',