Merged PR 3845699: [linkmgrd]: Introduce MUX cable linkmgrd

Linkmgrd monitors link status, mux status, and link state. Has
the link becomes unhealthy, linkmgrd will trigger mux switchover
on a standby ToR ensuring uninterrupted service to servers/blades.
This PR is initial implementation of linkmgrd.

Also, docker-mux container hold packages related to maintaining and managing
mux cable. It currently runs linkmgrd binary that monitor and switches
the mux if needed.
This PR also introduces mux-container and starts linkmgrd as startup when
build is configured with INCLUDE_MUX=y

Edit: linkmgrd PR will follow.

signed-off-by: Tamer Ahmed <tamer.ahmed@microsoft.com>

Related work items: #2315, #3146150

Makefile.work

@@ -10,6 +10,7 @@
 #  * ENABLE_ZTP: Enables zero touch provisioning.
 #  * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
 #  * INCLUDE_KUBERNETES: Allows including Kubernetes
+#  * INCLUDE_MUX: Include MUX feature/services for TOR switch.
 #  * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
 #  * by default for TOR switch.
 #  * ENABLE_SYNCD_RPC: Enables rpc-based syncd builds.
@@ -274,6 +275,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            INCLUDE_DHCP_RELAY=$(INCLUDE_DHCP_RELAY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
+                           SONIC_INCLUDE_MUX=$(INCLUDE_MUX) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \

dockers/docker-mux/Dockerfile.j2

@@ -0,0 +1,36 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-buster
+
+ARG docker_container_name
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update &&       \
+    apt-get install -f -y   \
+        libmnl0
+
+{% if docker_mux_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_mux_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_mux_debs.split(' ')) }}
+{%- endif %}
+
+## Clean up
+RUN apt-get clean -y        && \
+    apt-get autoclean -y    && \
+    apt-get autoremove -y   && \
+    rm -rf /debs
+
+COPY ["docker-init.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor/"]
+
+## Copy all Jinja2 template files into the templates folder
+COPY ["*.j2", "/usr/share/sonic/templates/"]
+
+ENTRYPOINT ["/usr/bin/docker-init.sh"]

dockers/docker-mux/critical_processes

@@ -0,0 +1 @@
+program:linkmgrd

dockers/docker-mux/docker-init.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Generate supervisord config file
+mkdir -p /etc/supervisor/conf.d/
+
+# The docker container should start this script as PID 1, so now that supervisord is
+# properly configured, we exec supervisord so that it runs as PID 1 for the
+# duration of the container's lifetime
+exec /usr/local/bin/supervisord

dockers/docker-mux/supervisord.conf

@@ -0,0 +1,41 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=100
+
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name mux
+events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
+autostart=true
+autorestart=unexpected
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:linkmgrd]
+command=nice -n -20 /usr/sbin/linkmgrd -v warning
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+startretries=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+

files/build_templates/init_cfg.json.j2

@@ -32,6 +32,7 @@
 {%- if sonic_asic_platform == "vs" %}{% do features.append(("gbsyncd", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_iccpd == "y" %}{% do features.append(("iccpd", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_mgmt_framework == "y" %}{% do features.append(("mgmt-framework", "enabled", true, "enabled")) %}{% endif %}
+{%- if include_mux == "y" %}{% do features.append(("mux", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_nat == "y" %}{% do features.append(("nat", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_restapi == "y" %}{% do features.append(("restapi", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_sflow == "y" %}{% do features.append(("sflow", "disabled", false, "enabled")) %}{% endif %}

files/build_templates/mux.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=MUX Cable Container
+Requires=database.service updategraph.service pmon.service swss.service
+After=pmon.service swss.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

rules/config

@@ -199,3 +199,6 @@ REGISTRY_SERVER ?= sonicdev-microsoft.azurecr.io
 
 # BUILD_MULTIASIC_KVM - if set to y multi-asic KVM images will be generated.
 BUILD_MULTIASIC_KVM = n
+
+# INCLUDE_MUX - build docker-mux for dual ToR (Gemini)
+INCLUDE_MUX = y

rules/docker-mux.dep

@@ -0,0 +1,12 @@
+
+DPATH       := $($(DOCKER_MUX)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-mux.mk rules/docker-mux.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_MUX)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_MUX)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_MUX)_DEP_FILES   := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_MUX),$(DOCKER_MUX_DBG)))
+

rules/docker-mux.mk

@@ -0,0 +1,32 @@
+# Docker image for MUX
+
+DOCKER_MUX_STEM = docker-mux
+DOCKER_MUX = $(DOCKER_MUX_STEM).gz
+DOCKER_MUX_DBG = $(DOCKER_MUX_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_MUX)_PATH = $(DOCKERS_PATH)/$(DOCKER_MUX_STEM)
+
+$(DOCKER_MUX)_DEPENDS = $(SONIC_LINKMGRD) $(LIBSWSSCOMMON) $(LIBHIREDIS)
+$(DOCKER_MUX)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS)
+$(DOCKER_MUX)_DBG_DEPENDS += $(SONIC_LINKMGRD_DBG) $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DBG)
+
+$(DOCKER_MUX)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
+
+$(DOCKER_MUX)_LOAD_DOCKERS = $(DOCKER_CONFIG_ENGINE_BUSTER)
+
+ifeq ($(INCLUDE_MUX), y)
+SONIC_DOCKER_IMAGES += $(DOCKER_MUX)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MUX)
+endif
+
+ifeq ($(INCLUDE_MUX), y)
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_MUX_DBG)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_MUX_DBG)
+endif
+
+$(DOCKER_MUX)_CONTAINER_NAME = mux
+$(DOCKER_MUX)_RUN_OPT += --privileged -t
+$(DOCKER_MUX)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_ORCHAGENT)_RUN_OPT += -v /var/log/mux:/var/log/mux:rw
+$(DOCKER_MUX)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+

rules/linkmgrd.dep

@@ -0,0 +1,11 @@
+
+SPATH       := $($(SONIC_LINKMGRD)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/linkmgrd.mk rules/linkmgrd.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_LINKMGRD)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_LINKMGRD)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_LINKMGRD)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_LINKMGRD)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_LINKMGRD)_SMDEP_PATHS := $(SPATH)

rules/linkmgrd.mk

@@ -0,0 +1,18 @@
+# SONiC LINK ManaGeR Daemon package
+
+SONIC_LINKMGRD_VERSION = 1.0.0-1
+SONIC_LINKMGRD_PKG_NAME = linkmgrd
+
+export SONIC_LINKMGRD_VERSION SONIC_LINKMGRD_PKG_NAME
+
+SONIC_LINKMGRD = sonic-$(SONIC_LINKMGRD_PKG_NAME)_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_LINKMGRD)_SRC_PATH = $(SRC_PATH)/$(SONIC_LINKMGRD_PKG_NAME)
+$(SONIC_LINKMGRD)_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON) $(LIBHIREDIS_DEV) $(LIBHIREDIS)
+
+SONIC_DPKG_DEBS += $(SONIC_LINKMGRD)
+
+SONIC_LINKMGRD_DBG = sonic-$(SONIC_LINKMGRD_PKG_NAME)-dbgsym_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_LINKMGRD)_DBG_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DEV) $(LIBHIREDIS_DBG)
+$(eval $(call add_derived_package,$(SONIC_LINKMGRD),$(SONIC_LINKMGRD_DBG)))
+
+export SONIC_LINKMGRD SONIC_LINKMGRD_DBG

slave.mk

@@ -160,6 +160,10 @@ ifeq ($(SONIC_INCLUDE_MACSEC),y)
 INCLUDE_MACSEC = y
 endif
 
+ifeq ($(SONIC_INCLUDE_MUX),y)
+INCLUDE_MUX = y
+endif
+
 include $(RULES_PATH)/functions
 
 ifeq ($(SONIC_USE_PDDF_FRAMEWORK),y)
@@ -281,6 +285,7 @@ $(info "INCLUDE_NAT"                     : "$(INCLUDE_NAT)")
 $(info "INCLUDE_DHCP_RELAY"              : "$(INCLUDE_DHCP_RELAY)")
 $(info "INCLUDE_KUBERNETES"              : "$(INCLUDE_KUBERNETES)")
 $(info "INCLUDE_MACSEC"                  : "$(INCLUDE_MACSEC)")
+$(info "INCLUDE_MUX"                     : "$(INCLUDE_MUX)")
 $(info "TELEMETRY_WRITABLE"              : "$(TELEMETRY_WRITABLE)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
 $(info "MULTIARCH_QEMU_ENVIRON"          : "$(MULTIARCH_QEMU_ENVIRON)")
@@ -1003,6 +1008,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export components="$(foreach component,$(notdir $^),\
 		$(shell [[ ! -z '$($(component)_VERSION)' && ! -z '$($(component)_NAME)' ]] && \
 			echo $($(component)_NAME)==$($(component)_VERSION)))"
+	export include_mux="$(INCLUDE_MUX)"
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"
 		export docker_image_name="$(basename $(docker))"