[sonic-host-service]: Add SONiC Host Services infrastructure (#4840)

- Why I did it

When SONiC is configured with the management framework and/or telemetry services, the applications running inside those containers need to access some functionality on the host system. The following is a non-exhaustive list of such functionality:

Image management
Configuration save and load
ZTP enable/disable and status
Show tech support
- How I did it

The host service is a Python process that listens for requests via D-Bus. It will then service those requests and send a response back to the requestor.

This PR only introduces the host service infrastructure. Applications that need access to the host services must add applets that will register on D-Bus endpoints to service the appropriate functionality.

- How to verify it

- Description for the changelog

Add SONiC Host Service for container to execute select commands in host

Signed-off-by: Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>

files/build_templates/sonic_debian_extension.j2

@@ -196,6 +196,12 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 {% endif %}
 
+{% if include_host_service == "y" %}
+# Install SONiC Host Service (and its dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-service_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+{% endif %}
+
 # SONiC utilities installs bash-completion as a dependency. However, it is disabled by default
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/

rules/config

@@ -125,6 +125,10 @@ INCLUDE_SFLOW = y
 # INCLUDE_MGMT_FRAMEWORK - build docker-sonic-mgmt-framework for CLI and REST server support
 INCLUDE_MGMT_FRAMEWORK = y
 
+# INCLUDE_HOST_SERVICE - build sonic-host-services for mgmt-framework and/or
+# telemetry containers to access host functionality
+INCLUDE_HOST_SERVICE = n
+
 # INCLUDE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
 INCLUDE_RESTAPI = n
 

rules/docker-sonic-mgmt-framework.mk

@@ -28,6 +28,7 @@ $(DOCKER_MGMT_FRAMEWORK)_CONTAINER_NAME = mgmt-framework
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --privileged -t
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc:/host_etc:ro
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
 
 $(DOCKER_MGMT_FRAMEWORK)_BASE_IMAGE_FILES += sonic-cli:/usr/bin/sonic-cli

rules/docker-telemetry.mk

@@ -26,6 +26,7 @@ endif
 $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry
 $(DOCKER_TELEMETRY)_RUN_OPT += --privileged -t
 $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_TELEMETRY)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
 $(DOCKER_TELEMETRY)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
 
 $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

rules/sonic-host-service.dep

@@ -0,0 +1,11 @@
+
+SPATH       := $($(SONIC_HOST_SERVICE)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-service.mk rules/sonic-host-service.dep
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_HOST_SERVICE)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICE)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICE)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_HOST_SERVICE)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_HOST_SERVICE)_SMDEP_PATHS := $(SPATH)
+

rules/sonic-host-service.mk

@@ -0,0 +1,9 @@
+# SONiC host service package
+
+ifeq ($(INCLUDE_HOST_SERVICE), y)
+
+SONIC_HOST_SERVICE = sonic-host-service_1.0.0_all.deb
+$(SONIC_HOST_SERVICE)_SRC_PATH = $(SRC_PATH)/sonic-host-service
+SONIC_MAKE_DEBS += $(SONIC_HOST_SERVICE)
+
+endif

slave.mk

@@ -224,6 +224,7 @@ $(info "VS_PREPARE_MEM"                  : "$(VS_PREPARE_MEM)")
 $(info "INCLUDE_MGMT_FRAMEWORK"          : "$(INCLUDE_MGMT_FRAMEWORK)")
 $(info "INCLUDE_ICCPD"                   : "$(INCLUDE_ICCPD)")
 $(info "INCLUDE_SYSTEM_TELEMETRY"        : "$(INCLUDE_SYSTEM_TELEMETRY)")
+$(info "INCLUDE_HOST_SERVICE"            : "$(INCLUDE_HOST_SERVICE)")
 $(info "INCLUDE_RESTAPI"                 : "$(INCLUDE_RESTAPI)")
 $(info "INCLUDE_SFLOW"                   : "$(INCLUDE_SFLOW)")
 $(info "INCLUDE_NAT"                     : "$(INCLUDE_NAT)")
@@ -795,6 +796,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
         $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
+        $(if $(findstring y,$(INCLUDE_HOST_SERVICE)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_HOST_SERVICE))) \
         $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
@@ -821,6 +823,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export enable_ztp="$(ENABLE_ZTP)"
 	export include_system_telemetry="$(INCLUDE_SYSTEM_TELEMETRY)"
 	export include_restapi="$(INCLUDE_RESTAPI)"
+	export include_host_service="$(INCLUDE_HOST_SERVICE)"
 	export include_nat="$(INCLUDE_NAT)"
 	export include_sflow="$(INCLUDE_SFLOW)"
 	export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"

src/sonic-host-service/Makefile

@@ -0,0 +1,41 @@
+################################################################################
+#                                                                              #
+#  Copyright 2020 Dell Inc.                                                    #
+#                                                                              #
+#  Licensed under the Apache License, Version 2.0 (the "License");             #
+#  you may not use this file except in compliance with the License.            #
+#  You may obtain a copy of the License at                                     #
+#                                                                              #
+#     http://www.apache.org/licenses/LICENSE-2.0                               #
+#                                                                              #
+#  Unless required by applicable law or agreed to in writing, software         #
+#  distributed under the License is distributed on an "AS IS" BASIS,           #
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.    #
+#  See the License for the specific language governing permissions and         #
+#  limitations under the License.                                              #
+#                                                                              #
+################################################################################
+
+TOPDIR := $(abspath .)
+MAIN_TARGET = sonic-host-service_1.0.0_all.deb
+INSTALL := /usr/bin/install
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	dpkg-buildpackage -us -uc -b
+	mv ../$(MAIN_TARGET) $(DEST)/
+
+SOURCES := $(wildcard '*.*') $(wildcard 'host_modules/*.py')
+install: $(SOURCES)
+	# Scripts for host service
+	$(INSTALL) -d $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+	$(INSTALL) -D $(TOPDIR)/sonic_host_server.py $(DESTDIR)/usr/lib/sonic_host_service
+	$(INSTALL) -D $(TOPDIR)/host_modules/*.py $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+
+	# D-Bus permissions configuration
+	$(INSTALL) -d $(DESTDIR)/etc/dbus-1/system.d
+	$(INSTALL) -D $(TOPDIR)/org.sonic.hostservice.conf $(DESTDIR)/etc/dbus-1/system.d
+
+	# systemd unit
+	$(INSTALL) -d $(DESTDIR)/lib/systemd/system
+	$(INSTALL) -D $(TOPDIR)/sonic-hostservice.service $(DESTDIR)/lib/systemd/system
+

src/sonic-host-service/debian/.gitignore

@@ -0,0 +1,4 @@
+.debhelper
+sonic-host-service.*
+sonic-host-service/
+files

src/sonic-host-service/debian/changelog

@@ -0,0 +1,5 @@
+sonic-host-service (1.0.0) UNRELEASED; urgency=low
+
+  * Initial release.
+
+ -- Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>  Mon, 22 Jun 2020 00:00:00 +0000

src/sonic-host-service/debian/compat

@@ -0,0 +1 @@
+9

src/sonic-host-service/debian/control

@@ -0,0 +1,15 @@
+Source: sonic-host-service
+Maintainer: Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>
+Build-Depends: debhelper (>= 8.0.0),
+               dh-systemd
+Vcs-Git: https://github.com/Azure/sonic-buildimage
+Homepage: https://github.com/Azure/SONiC/
+Standards-Version: 3.9.3
+Section: net
+
+Package: sonic-host-service
+Priority: extra
+Architecture: all
+Depends: python3-systemd, python3-dbus, python3-gi, ${misc:Depends}
+Description: SONiC Host Service
+

src/sonic-host-service/debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+%:
+	dh $@ --with systemd --parallel
+
+override_dh_auto_clean:
+override_dh_auto_test:
+override_dh_auto_build:
+override_dh_auto_install:
+	make install DESTDIR=debian/sonic-host-service

src/sonic-host-service/host_modules/host_service.py

@@ -0,0 +1,34 @@
+"""Base class for host modules"""
+
+import dbus.service
+import dbus
+
+BUS_NAME_BASE = 'org.SONiC.HostService'
+BUS_PATH = '/org/SONiC/HostService'
+
+def bus_name(mod_name):
+    """Return the bus name for the service"""
+    return BUS_NAME_BASE + '.' + mod_name
+
+def bus_path(mod_name):
+    """Return the bus path for the service"""
+    return BUS_PATH + '/' + mod_name
+
+method = dbus.service.method
+
+class HostService(dbus.service.Object):
+    """Service class for top level DBus endpoint"""
+    def __init__(self, mod_name):
+        self.bus = dbus.SystemBus()
+        self.bus_name = dbus.service.BusName(BUS_NAME_BASE, self.bus)
+        super(HostService, self).__init__(self.bus_name, BUS_PATH)
+
+class HostModule(dbus.service.Object):
+    """Base class for all host modules"""
+    def __init__(self, mod_name):
+        self.bus = dbus.SystemBus()
+        self.bus_name = dbus.service.BusName(bus_name(mod_name), self.bus)
+        super(HostModule, self).__init__(self.bus_name, bus_path(mod_name))
+
+def register():
+    return HostService, "host_service"

src/sonic-host-service/org.sonic.hostservice.conf

@@ -0,0 +1,18 @@
+<!DOCTYPE busconfig PUBLIC
+ "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
+ "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
+<busconfig>
+
+  <!-- Only root can own the bus -->
+
+  <policy user="root">
+    <allow own_prefix="org.SONiC.HostService"/>
+  </policy>
+
+  <!-- Allow user "root" to invoke methods on the bus -->
+  <policy user="root">
+    <allow send_destination="org.SONiC.HostService"/>
+    <allow receive_sender="org.SONiC.HostService"/>
+  </policy>
+
+</busconfig>

src/sonic-host-service/sonic-hostservice.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=SONiC Host Service
+
+[Service]
+Type=dbus
+BusName=org.SONiC.HostService
+
+ExecStart=/usr/bin/python3 -u /usr/lib/sonic_host_service/sonic_host_server.py
+
+Restart=on-failure
+RestartSec=10
+TimeoutStopSec=3
+
+[Install]
+WantedBy=multi-user.target

src/sonic-host-service/sonic_host_server.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Host Service to handle docker-to-host communication"""
+
+import os
+import os.path
+import glob
+import importlib
+import sys
+
+import dbus
+import dbus.service
+import dbus.mainloop.glib
+
+from gi.repository import GObject
+
+def register_modules():
+    """Register all host modules"""
+    mod_path = os.path.join(os.path.dirname(__file__), 'host_modules')
+    sys.path.append(mod_path)
+    for mod_file in glob.glob(os.path.join(mod_path, '*.py')):
+        if os.path.isfile(mod_file) and not mod_file.endswith('__init__.py'):
+            mod_name = os.path.basename(mod_file)[:-3]
+            module = importlib.import_module(mod_name)
+
+            register_cb = getattr(module, 'register', None)
+            if not register_cb:
+                raise Exception('Missing register function for ' + mod_name)
+
+            register_dbus(register_cb)
+
+def register_dbus(register_cb):
+    """Register DBus handlers for individual modules"""
+    handler_class, mod_name = register_cb()
+    handlers[mod_name] = handler_class(mod_name)
+
+# Create a main loop reactor
+GObject.threads_init()
+dbus.mainloop.glib.threads_init()
+dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+loop = GObject.MainLoop()
+handlers = {}
+
+class SignalManager(object):
+    ''' This is used to manage signals received (e.g. SIGINT).
+        When stopping a process (systemctl stop [service]), systemd sends
+        a SIGTERM signal.
+    '''
+    shutdown = False
+    def __init__(self):
+        ''' Install signal handlers.
+
+            SIGTERM is invoked when systemd wants to stop the daemon.
+            For example, "systemctl stop mydaemon.service"
+            or,          "systemctl restart mydaemon.service"
+
+        '''
+        import signal
+        signal.signal(signal.SIGTERM, self.sigterm_hdlr)
+
+    def sigterm_hdlr(self, _signum, _frame):
+        self.shutdown = True
+        loop.quit()
+
+sigmgr = SignalManager()
+register_modules()
+
+# Only run if we actually have some handlers
+if handlers:
+    import systemd.daemon
+    systemd.daemon.notify("READY=1")
+
+    while not sigmgr.shutdown:
+        loop.run()
+        if sigmgr.shutdown:
+            break
+
+    systemd.daemon.notify("STOPPING=1")
+else:
+    print("No handlers to register, quitting...")