diff --git a/dockers/docker-dhcp-relay/Dockerfile.j2 b/dockers/docker-dhcp-relay/Dockerfile.j2 index 58796ca79e..1100aa5107 100644 --- a/dockers/docker-dhcp-relay/Dockerfile.j2 +++ b/dockers/docker-dhcp-relay/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update diff --git a/dockers/docker-dhcp-relay/start.sh b/dockers/docker-dhcp-relay/start.sh index 86ba91147e..cb563eb003 100755 --- a/dockers/docker-dhcp-relay/start.sh +++ b/dockers/docker-dhcp-relay/start.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f dhcp_relay -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + # If our supervisor config has entries in the "isc-dhcp-relay" group... if [ $(supervisorctl status | grep -c "^isc-dhcp-relay:") -gt 0 ]; then # Wait for all interfaces to come up and be assigned IPv4 addresses before diff --git a/dockers/docker-lldp/Dockerfile.j2 b/dockers/docker-lldp/Dockerfile.j2 index 1306582b0a..297f776651 100644 --- a/dockers/docker-lldp/Dockerfile.j2 +++ b/dockers/docker-lldp/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update diff --git a/dockers/docker-lldp/start.sh b/dockers/docker-lldp/start.sh index 5cb6042cee..b2c82d6d04 100755 --- a/dockers/docker-lldp/start.sh +++ b/dockers/docker-lldp/start.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f lldp -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + sonic-cfggen -d -t /usr/share/sonic/templates/lldpd.conf.j2 > /etc/lldpd.conf mkdir -p /var/sonic diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2 index b45fbc84d9..1aed981db1 100755 --- a/dockers/docker-platform-monitor/Dockerfile.j2 +++ b/dockers/docker-platform-monitor/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Install required packages RUN apt-get update && \ apt-get install -y \ diff --git a/dockers/docker-platform-monitor/start.sh b/dockers/docker-platform-monitor/start.sh index 2f6dc61a84..98aa974e13 100755 --- a/dockers/docker-platform-monitor/start.sh +++ b/dockers/docker-platform-monitor/start.sh @@ -2,6 +2,16 @@ declare -r EXIT_SUCCESS="0" +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f pmon -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /var/sonic echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status diff --git a/dockers/docker-router-advertiser/Dockerfile.j2 b/dockers/docker-router-advertiser/Dockerfile.j2 index 3896286556..7d225cbd4e 100644 --- a/dockers/docker-router-advertiser/Dockerfile.j2 +++ b/dockers/docker-router-advertiser/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update @@ -27,6 +31,7 @@ RUN apt-get clean -y && \ apt-get autoremove -y && \ rm -rf /debs +COPY ["start.sh", "/usr/bin/"] COPY ["docker-init.sh", "/usr/bin/"] COPY ["radvd.conf.j2", "wait_for_link.sh.j2", "docker-router-advertiser.supervisord.conf.j2", "/usr/share/sonic/templates/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] diff --git a/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 b/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 index ae73cd6e98..ae48792285 100644 --- a/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 +++ b/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 @@ -27,6 +27,17 @@ stdout_logfile=syslog stderr_logfile=syslog dependent_startup=true +[program:start] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +startsecs=0 +stdout_logfile=syslog +stderr_logfile=syslog +dependent_startup=true +dependent_startup_wait_for=rsyslogd:running + {# Router advertiser should only run on ToR (T0) devices which have #} {# at least one VLAN interface which has an IPv6 address asigned #} {%- set vlan_v6 = namespace(count=0) -%} @@ -51,7 +62,7 @@ startsecs=0 stdout_logfile=syslog stderr_logfile=syslog dependent_startup=true -dependent_startup_wait_for=rsyslogd:running +dependent_startup_wait_for=start:exited [program:radvd] command=/usr/sbin/radvd -n diff --git a/dockers/docker-router-advertiser/start.sh b/dockers/docker-router-advertiser/start.sh new file mode 100755 index 0000000000..561cd5d7ce --- /dev/null +++ b/dockers/docker-router-advertiser/start.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f radv -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + diff --git a/dockers/docker-snmp/Dockerfile.j2 b/dockers/docker-snmp/Dockerfile.j2 index d479392bb2..2fdde61252 100644 --- a/dockers/docker-snmp/Dockerfile.j2 +++ b/dockers/docker-snmp/Dockerfile.j2 @@ -2,6 +2,7 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Enable -O for all Python calls @@ -10,6 +11,9 @@ ENV PYTHONOPTIMIZE 1 # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages # Install make/gcc which is required for installing hiredis RUN apt-get update && \ diff --git a/dockers/docker-snmp/start.sh b/dockers/docker-snmp/start.sh index 37ba3577e5..7b02c5c086 100755 --- a/dockers/docker-snmp/start.sh +++ b/dockers/docker-snmp/start.sh @@ -1,5 +1,16 @@ #!/usr/bin/env bash + +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f snmp -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /etc/ssw /etc/snmp SONIC_CFGGEN_ARGS=" \ diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index 65e436d293..88ff943182 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + RUN apt-get update {% if docker_sonic_telemetry_debs.strip() -%} diff --git a/dockers/docker-sonic-telemetry/start.sh b/dockers/docker-sonic-telemetry/start.sh index d6722a27fc..08f7292f55 100755 --- a/dockers/docker-sonic-telemetry/start.sh +++ b/dockers/docker-sonic-telemetry/start.sh @@ -1,4 +1,14 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f telemetry -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /var/sonic echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 9c7afa84b2..dd37506f37 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -210,13 +210,14 @@ start() { DOCKERMOUNT=`getMountPoint "$DOCKERCHECK"` {%- endif %} if [ x"$DOCKERMOUNT" == x"$MOUNTPATH" ]; then + preStartAction {%- if docker_container_name == "database" %} echo "Starting existing ${DOCKERNAME} container" + docker start ${DOCKERNAME} {%- else %} echo "Starting existing ${DOCKERNAME} container with HWSKU $HWSKU" + /usr/local/bin/container start ${DOCKERNAME} {%- endif %} - preStartAction - docker start ${DOCKERNAME} postStartAction exit $? fi @@ -343,6 +344,7 @@ start() { {%- endif %} docker create {{docker_image_run_opt}} \ --net=$NET \ + -e RUNTIME_OWNER=local \ --uts=host \{# W/A: this should be set per-docker, for those dockers which really need host's UTS namespace #} {%- if install_debug_image == "y" %} -v /src:/src:ro -v /debug:/debug:rw \ @@ -400,21 +402,31 @@ start() { } preStartAction + {%- if docker_container_name == "database" %} docker start $DOCKERNAME + {%- else %} + /usr/local/bin/container start ${DOCKERNAME} + {%- endif %} postStartAction } wait() { + {%- if docker_container_name == "database" %} docker wait $DOCKERNAME + {%- else %} + /usr/local/bin/container wait $DOCKERNAME + {%- endif %} } stop() { + {%- if docker_container_name == "database" %} docker stop $DOCKERNAME -{%- if docker_container_name == "database" %} if [ "$DEV" ]; then ip netns delete "$NET_NS" fi -{%- endif %} + {%- else %} + /usr/local/bin/container stop $DOCKERNAME + {%- endif %} } DOCKERNAME={{docker_container_name}} diff --git a/files/build_templates/init_cfg.json.j2 b/files/build_templates/init_cfg.json.j2 index 82f300e007..cb68dd82f3 100644 --- a/files/build_templates/init_cfg.json.j2 +++ b/files/build_templates/init_cfg.json.j2 @@ -44,6 +44,10 @@ "has_global_scope": {% if feature + '.service' in installer_services.split(' ') %}true{% else %}false{% endif %}, "has_per_asic_scope": {% if feature + '@.service' in installer_services.split(' ') %}true{% else %}false{% endif %}, "auto_restart": "{{autorestart}}", +{%- if include_kubernetes == "y" %} +{%- if feature in ["dhcp_relay", "lldp", "pmon", "radv", "snmp", "telemetry"] %} + "set_owner": "kube", {% else %} + "set_owner": "local", {% endif %} {% endif %} "high_mem_alert": "disabled" }{% if not loop.last %},{% endif -%} {% endfor %} diff --git a/files/build_templates/kube_cni.10-flannel.conflist b/files/build_templates/kube_cni.10-flannel.conflist new file mode 100644 index 0000000000..17b50fda12 --- /dev/null +++ b/files/build_templates/kube_cni.10-flannel.conflist @@ -0,0 +1,21 @@ +{ + "_comment": "This is a dummy file to simulate flannel so that node status will show as Ready", + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +} + diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index c346b8455f..1aa3e42fe7 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -37,6 +37,7 @@ FILESYSTEM_ROOT_USR="$FILESYSTEM_ROOT/usr" FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM="$FILESYSTEM_ROOT/usr/lib/systemd/system" FILESYSTEM_ROOT_USR_SHARE="$FILESYSTEM_ROOT_USR/share" FILESYSTEM_ROOT_USR_SHARE_SONIC="$FILESYSTEM_ROOT_USR_SHARE/sonic" +FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS="$FILESYSTEM_ROOT_USR_SHARE_SONIC/scripts" FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES="$FILESYSTEM_ROOT_USR_SHARE_SONIC/templates" FILESYSTEM_ROOT_ETC="$FILESYSTEM_ROOT/etc" FILESYSTEM_ROOT_ETC_SONIC="$FILESYSTEM_ROOT_ETC/sonic" @@ -395,10 +396,36 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install futures==3.3.0 {% if include_kubernetes == "y" %} -# Copy kubelet service files -# Keep it disabled until join, else it continuously restart and as well spew too many -# non-required log lines wasting syslog resources. +# Install remote Container mgmt package +# Required even if include_kubernetes != y, as it contains the +# the container wrapper for docker start/stop/wait commands. +# +SONIC_CTRMGMT_WHEEL_NAME=$(basename {{sonic_ctrmgmt_py3_wheel_path}}) +sudo cp {{sonic_ctrmgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_CTRMGMT_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_CTRMGMT_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_CTRMGMT_WHEEL_NAME + +# Copy remote container mangement files +# File called from each container upon start/stop to record the state +sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS} +sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/ +sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py + +# Config file used by container mgmt scripts/service +sudo cp ${files_path}/remote_ctr.config.json ${FILESYSTEM_ROOT_ETC_SONIC}/ + +# Remote container management service files +sudo cp ${files_path}/ctrmgrd.service ${FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM}/ +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ctrmgrd.service + +# kubelet service is controlled by ctrmgrd daemon. sudo LANG=C chroot $FILESYSTEM_ROOT systemctl disable kubelet.service +{% else %} +# container script for docker commands, which is required as +# all docker commands are replaced with container commands. +# So just copy that file only. +# +sudo cp ${files_path}/container $FILESYSTEM_ROOT/usr/local/bin/ {% endif %} # Copy the buffer configuration template @@ -585,6 +612,22 @@ else fi sudo umount /proc || true sudo rm $FILESYSTEM_ROOT/etc/init.d/docker + +sudo bash -c "echo { > $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% for entry in feature_vs_image_names.split(' ') -%} +{% if entry|length %} +{% set lst = entry.split(':') %} +{% set lst1 = lst[1].split('.') %} + sudo bash -c "echo -n -e \"\x22{{ lst[0] }}\x22 : \x22{{ lst1[0] }}\x22\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% if not loop.last %} + sudo bash -c "echo \",\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% else %} + sudo bash -c "echo \"\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% endif %} +{% endif %} +{% endfor %} +sudo bash -c "echo } >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" + {% for script in installer_start_scripts.split(' ') -%} sudo cp {{script}} $FILESYSTEM_ROOT/usr/bin/ {% endfor %} diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index 66ec301d1f..a3d037b9ca 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -12,6 +12,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) +$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BUSTER)_DBG_IMAGE_PACKAGES) diff --git a/rules/docker-dhcp-relay.mk b/rules/docker-dhcp-relay.mk index 860928bf81..6be9bb79fd 100644 --- a/rules/docker-dhcp-relay.mk +++ b/rules/docker-dhcp-relay.mk @@ -7,6 +7,7 @@ DOCKER_DHCP_RELAY_DBG = $(DOCKER_DHCP_RELAY_STEM)-$(DBG_IMAGE_MARK).gz $(DOCKER_DHCP_RELAY)_PATH = $(DOCKERS_PATH)/$(DOCKER_DHCP_RELAY_STEM) $(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_RELAY) $(SONIC_DHCPMON) + $(DOCKER_DHCP_RELAY)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS) $(DOCKER_DHCP_RELAY)_DBG_DEPENDS += $(ISC_DHCP_RELAY_DBG) @@ -23,4 +24,5 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_DHCP_RELAY_DBG) $(DOCKER_DHCP_RELAY)_CONTAINER_NAME = dhcp_relay $(DOCKER_DHCP_RELAY)_RUN_OPT += --privileged -t $(DOCKER_DHCP_RELAY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_DHCP_RELAY)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_DHCP_RELAY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/docker-lldp.mk b/rules/docker-lldp.mk index b84ff6d7e4..5cd4fd02a0 100644 --- a/rules/docker-lldp.mk +++ b/rules/docker-lldp.mk @@ -25,6 +25,7 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_LLDP_DBG) $(DOCKER_LLDP)_CONTAINER_NAME = lldp $(DOCKER_LLDP)_RUN_OPT += --privileged -t $(DOCKER_LLDP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_LLDP)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_LLDP)_BASE_IMAGE_FILES += lldpctl:/usr/bin/lldpctl $(DOCKER_LLDP)_BASE_IMAGE_FILES += lldpcli:/usr/bin/lldpcli diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk index e9e414cadd..1bd6a68660 100644 --- a/rules/docker-platform-monitor.mk +++ b/rules/docker-platform-monitor.mk @@ -45,6 +45,7 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_PLATFORM_MONITOR_DBG) $(DOCKER_PLATFORM_MONITOR)_CONTAINER_NAME = pmon $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += --privileged -t $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /var/run/platform_cache:/var/run/platform_cache:ro $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/device/pddf:/usr/share/sonic/device/pddf:ro diff --git a/rules/docker-router-advertiser.mk b/rules/docker-router-advertiser.mk index 3b9f7ae8ba..38bd959056 100644 --- a/rules/docker-router-advertiser.mk +++ b/rules/docker-router-advertiser.mk @@ -22,4 +22,5 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_ROUTER_ADVERTISER_DBG) $(DOCKER_ROUTER_ADVERTISER)_CONTAINER_NAME = radv $(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += --privileged -t $(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_ROUTER_ADVERTISER)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/docker-snmp.mk b/rules/docker-snmp.mk index 3493a2dfa3..cbc69dc92b 100644 --- a/rules/docker-snmp.mk +++ b/rules/docker-snmp.mk @@ -26,5 +26,6 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_SNMP_DBG) $(DOCKER_SNMP)_CONTAINER_NAME = snmp $(DOCKER_SNMP)_RUN_OPT += --privileged -t $(DOCKER_SNMP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_SNMP)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_SNMP)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) $(DOCKER_SNMP)_BASE_IMAGE_FILES += monit_snmp:/etc/monit/conf.d diff --git a/rules/docker-telemetry.mk b/rules/docker-telemetry.mk index 15f90c74c8..5152a38696 100644 --- a/rules/docker-telemetry.mk +++ b/rules/docker-telemetry.mk @@ -26,6 +26,7 @@ endif $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry $(DOCKER_TELEMETRY)_RUN_OPT += --privileged -t $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_TELEMETRY)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_TELEMETRY)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/sonic-ctrmgrd.dep b/rules/sonic-ctrmgrd.dep new file mode 100644 index 0000000000..b656421276 --- /dev/null +++ b/rules/sonic-ctrmgrd.dep @@ -0,0 +1,8 @@ +SPATH := $($(SONIC_CTRMGRD)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-ctrmgrd.mk rules/sonic-ctrmgrd.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(SONIC_CTRMGRD)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_CTRMGRD)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_CTRMGRD)_DEP_FILES := $(DEP_FILES) diff --git a/rules/sonic-ctrmgrd.mk b/rules/sonic-ctrmgrd.mk new file mode 100644 index 0000000000..659a2cf4ac --- /dev/null +++ b/rules/sonic-ctrmgrd.mk @@ -0,0 +1,31 @@ +# sonic-ctrmgrd package + +SONIC_CTRMGRD = sonic_ctrmgrd-1.0.0-py3-none-any.whl +$(SONIC_CTRMGRD)_SRC_PATH = $(SRC_PATH)/sonic-ctrmgrd +$(SONIC_CTRMGRD)_FILES_PATH = $($(SONIC_CTRMGRD)_SRC_PATH)/ctrmgr + +$(SONIC_CTRMGRD)_PYTHON_VERSION = 3 +$(SONIC_CTRMGRD)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON) +$(SONIC_CTRMGRD)_DEPENDS += $(SONIC_PY_COMMON_PY3) + +$(SONIC_CTRMGRD)_CONTAINER_SCRIPT = container +$($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_STARTUP_SCRIPT = container_startup.py +$($(SONIC_CTRMGRD)_STARTUP_SCRIPT)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_CFG_JSON = remote_ctr.config.json +$($(SONIC_CTRMGRD)_CFG_JSON)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_SERVICE = ctrmgrd.service +$($(SONIC_CTRMGRD)_SERVICE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +SONIC_PYTHON_WHEELS += $(SONIC_CTRMGRD) + +$(SONIC_CTRMGRD)_FILES = $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_CFG_JSON) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_SERVICE) + +SONIC_COPY_FILES += $($(SONIC_CTRMGRD)_FILES) + diff --git a/slave.mk b/slave.mk index 6bd66ff589..a7b3ae033d 100644 --- a/slave.mk +++ b/slave.mk @@ -750,6 +750,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ --build-arg frr_user_uid=$(FRR_USER_UID) \ --build-arg frr_user_gid=$(FRR_USER_GID) \ + --build-arg image_version=$(SONIC_IMAGE_VERSION) \ --label Tag=$(SONIC_IMAGE_VERSION) \ -t $* $($*.gz_PATH) $(LOG) scripts/collect_docker_version_files.sh $* $(TARGET_PATH) @@ -865,6 +866,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \ $(if $(findstring y,$(PDDF_SUPPORT)),$(addprefix $(PYTHON_WHEELS_PATH)/,$(PDDF_PLATFORM_API_BASE_PY2))) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CTRMGRD)) \ + $(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \ @@ -911,6 +914,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))" export install_debug_image="$(INSTALL_DEBUG_TOOLS)" export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))" + export sonic_ctrmgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CTRMGRD))" export sonic_yang_mgmt_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))" export sonic_yang_mgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3))" export multi_instance="false" @@ -959,6 +963,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ # Exported variables are used by sonic_debian_extension.sh export installer_start_scripts="$(foreach docker, $($*_DOCKERS),$(addsuffix .sh, $($(docker:-dbg.gz=.gz)_CONTAINER_NAME)))" + export feature_vs_image_names="$(foreach docker, $($*_DOCKERS), $(addsuffix :, $($(docker:-dbg.gz=.gz)_CONTAINER_NAME):$(docker:-dbg.gz=.gz)))" # Marks template services with an "@" according to systemd convention # If the $($docker)_TEMPLATE) variable is set, the service will be treated as a template diff --git a/src/sonic-ctrmgrd/.gitignore b/src/sonic-ctrmgrd/.gitignore new file mode 100644 index 0000000000..bdebd5e838 --- /dev/null +++ b/src/sonic-ctrmgrd/.gitignore @@ -0,0 +1,12 @@ +.eggs/ +build/ +dist/ +*.egg-info/ +ctrmgr/*.pyc +tests/*.pyc +tests/__pycache__/ +.idea +.coverage +ctrmgr/__pycache__/ +venv +tests/.coverage* diff --git a/src/sonic-ctrmgrd/README.rst b/src/sonic-ctrmgrd/README.rst new file mode 100644 index 0000000000..30207f44b3 --- /dev/null +++ b/src/sonic-ctrmgrd/README.rst @@ -0,0 +1,3 @@ +" +This Package contains modules required for remote container management. +" diff --git a/src/sonic-ctrmgrd/ctrmgr/__init__.py b/src/sonic-ctrmgrd/ctrmgr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sonic-ctrmgrd/ctrmgr/container b/src/sonic-ctrmgrd/ctrmgr/container new file mode 100755 index 0000000000..ca0963946f --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/container @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 + +import argparse +import os +import inspect +import json +import syslog +import time +import datetime + +import docker +from swsscommon import swsscommon + +CTR_STATE_SCR_PATH = '/usr/share/sonic/scripts/container_startup.py' + +state_db = None + +# DB field names +FEATURE_TABLE = "FEATURE" +SET_OWNER = "set_owner" +NO_FALLBACK = "no_fallback_to_local" + +CURRENT_OWNER = "current_owner" +UPD_TIMESTAMP = "update_time" +CONTAINER_ID = "container_id" +REMOTE_STATE = "remote_state" +VERSION = "container_version" +SYSTEM_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +# Get seconds to wait for remote docker to start. +# If not, revert to local +# +SONIC_CTR_CONFIG = "/etc/sonic/remote_ctr.config.json" +SONIC_CTR_CONFIG_PEND_SECS = "revert_to_local_on_wait_seconds" +DEFAULT_PEND_SECS = ( 5 * 60 ) +WAIT_POLL_SECS = 2 + +remote_ctr_enabled = False + +def debug_msg(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + # print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def init(): + """ Get DB connections """ + global state_db, cfg_db, remote_ctr_enabled + + cfg_db = swsscommon.DBConnector("CONFIG_DB", 0) + state_db = swsscommon.DBConnector("STATE_DB", 0) + + remote_ctr_enabled = os.path.exists(CTR_STATE_SCR_PATH) + + +def get_config_data(fld, dflt): + """ Read entry from kube config file """ + if os.path.exists(SONIC_CTR_CONFIG): + with open(SONIC_CTR_CONFIG, "r") as s: + d = json.load(s) + if fld in d: + return d[fld] + return dflt + + +def read_data(is_config, feature, fields): + """ Read data from DB for desired fields using given defaults""" + ret = [] + + db = cfg_db if is_config else state_db + + tbl = swsscommon.Table(db, FEATURE_TABLE) + + data = dict(tbl.get(feature)[1]) + for (field, default) in fields: + val = data.get(field, default) + ret += [val] + + debug_msg("config:{} feature:{} fields:{} val:{}".format( + is_config, feature, str(fields), str(ret))) + + return tuple(ret) + + +def read_config(feature): + """ Read requried feature config """ + set_owner, no_fallback = read_data(True, feature, + [(SET_OWNER, "local"), (NO_FALLBACK, False)]) + + return (set_owner, not no_fallback) + + +def read_state(feature): + """ Read requried feature state """ + + return read_data(False, feature, + [(CURRENT_OWNER, "none"), (REMOTE_STATE, "none"), (CONTAINER_ID, "")]) + + +def docker_action(action, feature): + """ Execute docker action """ + try: + client = docker.from_env() + container = client.containers.get(feature) + getattr(container, action)() + syslog.syslog(syslog.LOG_INFO, "docker cmd: {} for {}".format(action, feature)) + return 0 + + except (docker.errors.NotFound, docker.errors.APIError) as err: + syslog.syslog(syslog.LOG_ERR, "docker cmd: {} for {} failed with {}". + format(action, feature, str(err))) + return -1 + + +def set_label(feature, create): + """ Set/drop label as required + Update is done in state-db. + ctrmgrd sets it with kube API server as required + """ + if remote_ctr_enabled: + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + fld = "{}_enabled".format(feature) + + # redundant set (data already exist) can still raise subscriber + # notification. So check & set. + # Redundant delete (data doesn't exist) does not raise any + # subscriber notification. So no need to pre-check for delete. + # + tbl.set(KUBE_LABEL_SET_KEY, [(fld, "true" if create else "false")]) + + +def update_data(feature, data): + if remote_ctr_enabled: + debug_msg("feature:{} data:{}".format(feature, str(data))) + tbl = swsscommon.Table(state_db, FEATURE_TABLE) + tbl.set(feature, list(data.items())) + + +def container_id(feature): + """ + Return the container ID for the feature. + + if current_owner is local, use feature name as the start/stop + of local image is synchronous. + Else get it from FEATURE table in STATE-DB + + :param feature: Name of the feature to start. + + """ + init() + + tbl = swsscommon.Table(state_db, "FEATURE") + data = dict(tbl.get(feature)[1]) + + if (data.get(CURRENT_OWNER, "").lower() == "local"): + return feature + else: + return data.get(CONTAINER_ID, feature) + + +def container_start(feature): + """ + Starts a container for given feature. + + Starts from local image and/or trigger kubernetes to deploy the image + for this feature. Marks the feature state up in STATE-DB FEATURE table. + + If feature's set_owner is local, invoke docker start. + If feature's set_owner is kube, it creates a node label that + would trigger kubernetes to start the container. With kube as + owner, if fallback is enabled and remote_state==none, it starts + the local image using docker, which will run until kube + deployment occurs. + + :param feature: Name of the feature to start. + + """ + START_LOCAL = 1 + START_KUBE = 2 + + ret = 0 + debug_msg("BEGIN") + + init() + + set_owner, fallback = read_config(feature) + _, remote_state, _ = read_state(feature) + + debug_msg("{}: set_owner:{} fallback:{} remote_state:{}".format( + feature, set_owner, fallback, remote_state)) + + data = { + SYSTEM_STATE: "up", + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + } + + + start_val = 0 + if set_owner == "local": + start_val = START_LOCAL + else: + start_val = START_KUBE + if fallback and (remote_state == "none"): + start_val |= START_LOCAL + + if start_val == START_LOCAL: + # Implies *only* local. + # Ensure label is not there, to block kube deployment. + set_label(feature, False) + data[REMOTE_STATE] = "none" + + if (start_val & START_LOCAL): + data[CURRENT_OWNER] = "local" + data[CONTAINER_ID] = feature + + update_data(feature, data) + + if (start_val & START_LOCAL): + ret = docker_action("start", feature) + + if (start_val & START_KUBE): + set_label(feature, True) + debug_msg("END") + return ret + + +def container_stop(feature): + """ + Stops the running container for this feature. + + Instruct/ensure kube terminates, by removing label, unless + an kube upgrade is happening. + + Gets the container ID for this feature and call docker stop. + + Marks the feature state down in STATE-DB FEATURE table. + + :param feature: Name of the feature to stop. + + """ + debug_msg("BEGIN") + + init() + + set_owner, _ = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + remove_label = (remote_state != "pending") or (set_owner == "local") + + debug_msg("{}: set_owner:{} current_owner:{} remote_state:{} docker_id:{}".format( + feature, set_owner, current_owner, remote_state, docker_id)) + + if remove_label: + set_label(feature, False) + + if docker_id: + docker_action("stop", docker_id) + else: + syslog.syslog( + syslog.LOG_ERR if current_owner != "none" else syslog.LOG_INFO, + "docker stop skipped as no docker-id for {}".format(feature)) + + # Container could get killed or crashed. In either case + # it does not have opportunity to mark itself down. + # Even during normal termination, with SIGTERM received + # container process may not have enough window of time to + # mark itself down and has the potential to get aborted. + # + # systemctl ensures that it handles only one instance for + # a feature at anytime and however the feature container + # exits, upon stop/kill/crash, systemctl-stop process + # is assured to get called. So mark the feature down here. + # + data = { + CURRENT_OWNER: "none", + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + CONTAINER_ID: "", + VERSION: "", + SYSTEM_STATE: "down" + } + if remote_state == "running": + data[REMOTE_STATE] = "stopped" + + update_data(feature, data) + + debug_msg("END") + + +def container_kill(feature): + """ + Kills the running container for this feature. + + Instruct/ensure kube terminates, by removing label. + + :param feature: Name of the feature to kill. + + """ + debug_msg("BEGIN") + + init() + + set_owner, _ = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + remove_label = (set_owner != "local") or (current_owner != "local") + + debug_msg("{}: set_owner:{} current_owner:{} remote_state:{} docker_id:{}".format( + feature, set_owner, current_owner, remote_state, docker_id)) + + if remove_label: + set_label(feature, False) + + if docker_id: + docker_action("kill", docker_id) + + else: + syslog.syslog( + syslog.LOG_ERR if current_owner != "none" else syslog.LOG_INFO, + "docker stop skipped as no docker-id for {}".format(feature)) + + + debug_msg("END") + + +def container_wait(feature): + """ + Waits on the running container for this feature. + + Get the container-id and call docker wait. + + If docker-id can't be obtained for a configurable fail-duration + the wait clears the feature's remote-state in STATE-DB FEATURE + table and exit. + + :param feature: Name of the feature to wait. + + """ + debug_msg("BEGIN") + + init() + + set_owner, fallback = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + pend_wait_secs = 0 + + if not docker_id and fallback: + pend_wait_secs = get_config_data( + SONIC_CTR_CONFIG_PEND_SECS, DEFAULT_PEND_SECS) + + debug_msg("{}: set_owner:{} ct_owner:{} state:{} id:{} pend={}".format( + feature, set_owner, current_owner, remote_state, docker_id, + pend_wait_secs)) + + while not docker_id: + if fallback: + pend_wait_secs = pend_wait_secs - WAIT_POLL_SECS + if pend_wait_secs < 0: + break + + time.sleep(WAIT_POLL_SECS) + + current_owner, remote_state, docker_id = read_state(feature) + + debug_msg("wait_loop: {} = {} {} {}".format(feature, current_owner, remote_state, docker_id)) + + if (remote_state == "pending"): + update_data(feature, {REMOTE_STATE: "ready"}) + + if not docker_id: + # Clear remote state and exit. + # systemd would restart and fallback to local + update_data(feature, { REMOTE_STATE: "none" }) + debug_msg("{}: Exiting to fallback as remote is *not* starting". + format(feature)) + else: + debug_msg("END -- transitioning to docker wait") + docker_action("wait", docker_id) + + +def main(): + parser=argparse.ArgumentParser(description="container commands for start/stop/wait/kill/id") + parser.add_argument("action", choices=["start", "stop", "wait", "kill", "id"]) + parser.add_argument("name") + + args = parser.parse_args() + + if args.action == "start": + container_start(args.name) + + elif args.action == "stop": + container_stop(args.name) + + elif args.action == "kill": + container_kill(args.name) + + elif args.action == "wait": + container_wait(args.name) + + elif args.action == "id": + id = container_id(args.name) + print(id) + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/container_startup.py b/src/sonic-ctrmgrd/ctrmgr/container_startup.py new file mode 100755 index 0000000000..c56160aa48 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/container_startup.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import inspect +import json +import subprocess +import syslog +import time + +from swsscommon import swsscommon + +# DB field names +SET_OWNER = "set_owner" + +CURRENT_OWNER = "current_owner" +UPD_TIMESTAMP = "update_time" +DOCKER_ID = "container_id" +REMOTE_STATE = "remote_state" +VERSION = "container_version" +SYSTEM_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +UNIT_TESTING = 0 + + +def debug_msg(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def _get_version_key(feature, version): + # Coin label for version control + return "{}_{}_enabled".format(feature, version) + + +def read_data(feature): + state_data = { + CURRENT_OWNER: "none", + UPD_TIMESTAMP: "", + DOCKER_ID: "", + REMOTE_STATE: "none", + VERSION: "0.0.0", + SYSTEM_STATE: "" + } + + set_owner = "local" + + # read owner from config-db and current state data from state-db. + db = swsscommon.DBConnector("CONFIG_DB", 0) + tbl = swsscommon.Table(db, 'FEATURE') + data = dict(tbl.get(feature)[1]) + + if (SET_OWNER in data): + set_owner = data[SET_OWNER] + + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + state_data.update(dict(tbl.get(feature)[1])) + + return (state_db, set_owner, state_data) + + +def read_fields(state_db, feature, fields): + # Read directly from STATE-DB, given fields + # for given feature. + # Fields is a list of tuples (, ) + # + tbl = swsscommon.Table(state_db, 'FEATURE') + ret = [] + + # tbl.get for non-existing feature would return + # [False, {} ] + # + data = dict(tbl.get(feature)[1]) + for (field, default) in fields: + val = data[field] if field in data else default + ret += [val] + + return tuple(ret) + + +def check_version_blocked(state_db, feature, version): + # Ensure this version is *not* blocked explicitly. + # + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + labels = dict(tbl.get(KUBE_LABEL_SET_KEY)[1]) + key = _get_version_key(feature, version) + return (key in labels) and (labels[key].lower() == "false") + + +def drop_label(state_db, feature, version): + # Mark given feature version as dropped in labels. + # Update is done in state-db. + # ctrmgrd sets it with kube API server per reaschability + + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + name = _get_version_key(feature, version) + tbl.set(KUBE_LABEL_SET_KEY, [ (name, "false")]) + + +def update_data(state_db, feature, data): + # Update STATE-DB entry for this feature with given data + # + debug_msg("{}: {}".format(feature, str(data))) + tbl = swsscommon.Table(state_db, "FEATURE") + tbl.set(feature, list(data.items())) + + +def get_docker_id(): + # Read the container-id + # Note: This script runs inside the context of container + # + cmd = 'cat /proc/self/cgroup | grep -e ":memory:" | rev | cut -f1 -d\'/\' | rev' + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = proc.communicate()[0].decode("utf-8") + return output.strip()[:12] + + +def instance_higher(feature, ct_version, version): + # Compares given version against current version in STATE-DB. + # Return True if this version is higher than current. + # + ret = False + if ct_version: + ct = ct_version.split('.') + nxt = version.split('.') + for cs, ns in zip(ct, nxt): + c = int(cs) + n = int(ns) + if n < c: + break + elif n > c: + ret = True + break + else: + # Empty version implies no one is running. + ret = True + + debug_msg("compare version: new:{} current:{} res={}".format( + version, ct_version, ret)) + return ret + + +def is_active(feature, system_state): + # Check current system state of the feature + if system_state == "up": + return True + else: + syslog.syslog(syslog.LOG_ERR, "Found inactive for {}".format(feature)) + return False + + +def update_state(state_db, feature, owner=None, version=None): + """ + sets owner, version & container-id for this feature in state-db. + + If owner is local, update label to block remote deploying same version or + if kube, sets state to "running". + + """ + data = { + CURRENT_OWNER: owner, + DOCKER_ID: get_docker_id() if owner != "local" else feature, + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + "hello": "world", + VERSION: version + } + + if (owner == "local"): + # Disable deployment of this version as available locally + drop_label(state_db, feature, version) + else: + data[REMOTE_STATE] = "running" + + debug_msg("{} up data:{}".format(feature, str(data))) + update_data(state_db, feature, data) + + +def do_freeze(feat, m): + # Exiting will kick off the container to run. + # So sleep forever with periodic logs. + # + while True: + syslog.syslog(syslog.LOG_ERR, "Blocking .... feat:{} docker_id:{} msg:{}".format( + feat, get_docker_id(), m)) + if UNIT_TESTING: + break + time.sleep(60) + + +def container_up(feature, owner, version): + """ + This is called by container upon post start. + + The container will run its application, only upon this call + complete. + + This call does the basic check for if this starting-container can be allowed + to run based on current state, and owner & version of this starting + container. + + If allowed to proceed, this info is recorded in state-db and return + to enable container start the main application. Else it proceeds to + sleep forever, blocking the container from starting the main application. + + """ + debug_msg("BEGIN") + (state_db, set_owner, state_data) = read_data(feature) + + debug_msg("args: feature={}, owner={}, version={} DB: set_owner={} state_data={}".format( + feature, owner, version, set_owner, json.dumps(state_data, indent=4))) + + if owner == "local": + update_state(state_db, feature, owner, version) + else: + if (set_owner == "local"): + do_freeze(feature, "bail out as set_owner is local") + return + + if not is_active(feature, state_data[SYSTEM_STATE]): + do_freeze(feature, "bail out as system state not active") + return + + if check_version_blocked(state_db, feature, version): + do_freeze(feature, "This version is marked disabled. Exiting ...") + return + + if not instance_higher(feature, state_data[VERSION], version): + # TODO: May Remove label __enabled + # Else kubelet will continue to re-deploy every 5 mins, until + # master removes the lable to un-deploy. + # + do_freeze(feature, "bail out as current deploy version {} is not higher". + format(version)) + return + + update_data(state_db, feature, { VERSION: version }) + + mode = state_data[REMOTE_STATE] + if mode in ("none", "running", "stopped"): + update_data(state_db, feature, { REMOTE_STATE: "pending" }) + mode = "pending" + else: + debug_msg("{}: Skip remote_state({}) update".format(feature, mode)) + + + i = 0 + while (mode != "ready"): + if (i % 10) == 0: + debug_msg("{}: remote_state={}. Waiting to go ready".format(feature, mode)) + i += 1 + + time.sleep(2) + mode, db_version = read_fields(state_db, + feature, [(REMOTE_STATE, "none"), (VERSION, "")]) + if version != db_version: + # looks like another instance has overwritten. Exit for now. + # If this happens to be higher version, next deploy by kube will fix + # This is a very rare window of opportunity, for this version to be higher. + # + do_freeze(feature, + "bail out as current deploy version={} is different than {}. re-deploy higher one". + format(version, db_version)) + return + if UNIT_TESTING: + return + + + update_state(state_db, feature, owner, version) + + debug_msg("END") + + +def main(): + parser = argparse.ArgumentParser(description="container_startup kube/local []") + + parser.add_argument("-f", "--feature", required=True) + parser.add_argument("-o", "--owner", choices=["local", "kube"], required=True) + parser.add_argument("-v", "--version", required=True) + + args = parser.parse_args() + container_up(args.feature, args.owner, args.version) + + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py new file mode 100755 index 0000000000..d85f76c4f9 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py @@ -0,0 +1,147 @@ +#! /usr/bin/env python3 + +import argparse +import json +import os +import syslog + +import docker +from swsscommon import swsscommon + +CTR_NAMES_FILE = "/usr/share/sonic/templates/ctr_image_names.json" + +LATEST_TAG = "latest" + +# DB fields +CT_OWNER = 'current_owner' +CT_ID = 'container_id' +SYSTEM_STATE = 'system_state' + +def _get_local_image_name(feature): + d = {} + if os.path.exists(CTR_NAMES_FILE): + with open(CTR_NAMES_FILE, "r") as s: + d = json.load(s) + return d[feature] if feature in d else None + + +def _remove_container(client, feature): + try: + # Remove stopped container instance, if any for this feature + container = client.containers.get(feature) + container.remove() + syslog.syslog(syslog.LOG_INFO, "Removed container for {}". + format(feature)) + return 0 + except Exception as err: + syslog.syslog(syslog.LOG_INFO, "No container to remove for {} err={}". + format(feature, str(err))) + return -1 + + +def _tag_container_image(feature, container_id, image_name, image_tag): + client = docker.from_env() + + try: + container = client.containers.get(container_id) + + # Tag this image for given name & tag + container.image.tag(image_name, image_tag, force=True) + + syslog.syslog(syslog.LOG_INFO, + "Tagged image for {} with container-id={} to {}:{}". + format(feature, container_id, image_name, image_tag)) + ret = _remove_container(client, feature) + return ret + + except Exception as err: + syslog.syslog(syslog.LOG_ERR, "Image tag: container:{} {}:{} failed with {}". + format(container_id, image_name, image_tag, str(err))) + return -1 + + +def tag_feature(feature=None, image_name=None, image_tag=None): + ret = 0 + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + keys = tbl.getKeys() + for k in keys: + if (not feature) or (k == feature): + d = dict(tbl.get(k)[1]) + owner = d.get(CT_OWNER, "") + id = d.get(CT_ID, "") + if not image_name: + image_name = _get_local_image_name(k) + if not image_tag: + image_tag = LATEST_TAG + if id and (owner == "kube") and image_name: + ret = _tag_container_image(k, id, image_name, image_tag) + else: + syslog.syslog(syslog.LOG_ERR, + "Skip to tag feature={} image={} tag={}".format( + k, str(image_name), str(image_tag))) + ret = -1 + + image_name=None + return ret + + +def func_tag_feature(args): + return tag_feature(args.feature, args.image_name, args.image_tag) + + +def func_tag_all(args): + return tag_feature() + + +def func_kill_all(args): + client = docker.from_env() + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + + keys = tbl.getKeys() + for k in keys: + data = dict(tbl.get(k)[1]) + is_up = data.get(SYSTEM_STATE, "").lower() == "up" + id = data.get(CT_ID, "") if is_up else "" + if id: + try: + container = client.containers.get(id) + container.kill() + syslog.syslog(syslog.LOG_INFO, + "Killed container for {} with id={}".format(k, id)) + except Exception as err: + syslog.syslog(syslog.LOG_ERR, + "kill: feature={} id={} unable to get container". + format(k, id)) + return -1 + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Remote container mgmt tools") + subparsers = parser.add_subparsers(title="ctrmgr_tools") + + parser_tag = subparsers.add_parser('tag') + parser_tag.add_argument("-f", "--feature", required=True) + parser_tag.add_argument("-n", "--image-name") + parser_tag.add_argument("-t", "--image-tag") + parser_tag.set_defaults(func=func_tag_feature) + + parser_tag_all = subparsers.add_parser('tag-all') + parser_tag_all.set_defaults(func=func_tag_all) + + parser_kill_all = subparsers.add_parser('kill-all') + parser_kill_all.set_defaults(func=func_kill_all) + + args = parser.parse_args() + if len(args.__dict__) < 1: + parser.print_help() + return -1 + + ret = args.func(args) + return ret + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py new file mode 100755 index 0000000000..ba4f0057bd --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 + +import datetime +import inspect +import json +import os +import sys +import syslog + +from collections import defaultdict + +from swsscommon import swsscommon +from sonic_py_common import device_info + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) +import kube_commands + +UNIT_TESTING = 0 +UNIT_TESTING_ACTIVE = 0 + +# Kube config file +SONIC_CTR_CONFIG = "/etc/sonic/remote_ctr.config.json" + +CONFIG_DB_NAME = "CONFIG_DB" +STATE_DB_NAME = "STATE_DB" + +# DB SERVER +SERVER_TABLE = "KUBERNETES_MASTER" +SERVER_KEY = "SERVER" +CFG_SER_IP = "ip" +CFG_SER_PORT = "port" +CFG_SER_DISABLE = "disable" +CFG_SER_INSECURE = "insecure" + +ST_SER_IP = "ip" +ST_SER_PORT = "port" +ST_SER_CONNECTED = "connected" +ST_SER_UPDATE_TS = "update_time" + +# DB FEATURE +FEATURE_TABLE = "FEATURE" +CFG_FEAT_OWNER = "set_owner" +CFG_FEAT_NO_FALLBACK = "no_fallback_to_local" +CFG_FEAT_FAIL_DETECTION = "remote_fail_detection" + +ST_FEAT_OWNER = "current_owner" +ST_FEAT_UPDATE_TS = "update_time" +ST_FEAT_CTR_ID = "container_id" +ST_FEAT_CTR_VER = "container_version" +ST_FEAT_REMOTE_STATE = "remote_state" +ST_FEAT_SYS_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +remote_connected = False + +dflt_cfg_ser = { + CFG_SER_IP: "", + CFG_SER_PORT: "6443", + CFG_SER_DISABLE: "false", + CFG_SER_INSECURE: "false" + } + +dflt_st_ser = { + ST_SER_IP: "", + ST_SER_PORT: "", + ST_SER_CONNECTED: "", + ST_SER_UPDATE_TS: "" + } + +dflt_cfg_feat= { + CFG_FEAT_OWNER: "local", + CFG_FEAT_NO_FALLBACK: "false", + CFG_FEAT_FAIL_DETECTION: "300" + } + +dflt_st_feat= { + ST_FEAT_OWNER: "none", + ST_FEAT_UPDATE_TS: "", + ST_FEAT_CTR_ID: "", + ST_FEAT_CTR_VER: "", + ST_FEAT_REMOTE_STATE: "none", + ST_FEAT_SYS_STATE: "" + } + +JOIN_LATENCY = "join_latency_on_boot_seconds" +JOIN_RETRY = "retry_join_interval_seconds" +LABEL_RETRY = "retry_labels_update_seconds" + +remote_ctr_config = { + JOIN_LATENCY: 10, + JOIN_RETRY: 10, + LABEL_RETRY: 2 + } + +def log_debug(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def log_error(m): + syslog.syslog(syslog.LOG_ERR, msg) + + +def log_info(m): + syslog.syslog(syslog.LOG_INFO, msg) + + +def ts_now(): + return str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + +def is_systemd_active(feat): + if not UNIT_TESTING: + status = os.system('systemctl is-active --quiet {}'.format(feat)) + else: + status = UNIT_TESTING_ACTIVE + log_debug("system status for {}: {}".format(feat, str(status))) + return status == 0 + + +def restart_systemd_service(server, feat, owner): + log_debug("Restart service {} to owner:{}".format(feat, owner)) + if not UNIT_TESTING: + status = os.system("systemctl restart {}".format(feat)) + else: + server.mod_db_entry(STATE_DB_NAME, + FEATURE_TABLE, feat, {"restart": "true"}) + status = 0 + if status != 0: + syslog.syslog(syslog.LOG_ERR, + "Failed to restart {} to switch to {}".format(feat, owner)) + else: + syslog.syslog(syslog.LOG_INFO, + "Restarted {} to switch to {}".format(feat, owner)) + return status + + +def init(): + if os.path.exists(SONIC_CTR_CONFIG): + with open(SONIC_CTR_CONFIG, "r") as s: + d = json.load(s) + remote_ctr_config.update(d) + + +class MainServer: + """ Implements main io-loop of the application + Accept handler registration per db/table. + Call handler on update + """ + + SELECT_TIMEOUT = 1000 + + def __init__(self): + """ Constructor """ + self.db_connectors = {} + self.selector = swsscommon.Select() + self.callbacks = defaultdict(lambda: defaultdict(list)) # db -> table -> handlers[] + self.timer_handlers = defaultdict(list) + self.subscribers = set() + + def register_db(self, db_name): + """ Get DB connector, if not there """ + if db_name not in self.db_connectors: + self.db_connectors[db_name] = swsscommon.DBConnector(db_name, 0) + + + def register_timer(self, ts, handler): + """ Register timer based handler. + The handler will be called on/after give timestamp, ts + """ + self.timer_handlers[ts].append(handler) + + + def register_handler(self, db_name, table_name, handler): + """ + Handler registration for any update in given table + in given db. The handler will be called for any update + to the table in this db. + """ + self.register_db(db_name) + + if table_name not in self.callbacks[db_name]: + conn = self.db_connectors[db_name] + subscriber = swsscommon.SubscriberStateTable(conn, table_name) + self.subscribers.add(subscriber) + self.selector.addSelectable(subscriber) + self.callbacks[db_name][table_name].append(handler) + + + def get_db_entry(self, db_name, table_name, key): + """ Return empty dict if key not present """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + return dict(tbl.get(key)[1]) + + + def mod_db_entry(self, db_name, table_name, key, data): + """ Modify entry for given table|key with given dict type data """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + print("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data))) + tbl.set(key, list(data.items())) + + + def set_db_entry(self, db_name, table_name, key, data): + """ Set given data as complete data, which includes + removing any fields that are in DB but not in data + """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + ct_data = dict(tbl.get(key)[1]) + for k in ct_data: + if k not in data: + # Drop fields that are not in data + tbl.hdel(key, k) + tbl.set(key, list(data.items())) + + + def run(self): + """ Main loop """ + while True: + timeout = MainServer.SELECT_TIMEOUT + ct_ts = datetime.datetime.now() + while self.timer_handlers: + k = sorted(self.timer_handlers.keys())[0] + if k <= ct_ts: + lst = self.timer_handlers[k] + del self.timer_handlers[k] + for fn in lst: + fn() + else: + timeout = (k - ct_ts).seconds + break + + state, _ = self.selector.select(timeout) + if state == self.selector.TIMEOUT: + continue + elif state == self.selector.ERROR: + if not UNIT_TESTING: + raise Exception("Received error from select") + else: + print("Skipped Exception; Received error from select") + return + + for subscriber in self.subscribers: + key, op, fvs = subscriber.pop() + if not key: + continue + log_debug("Received message : '%s'" % str((key, op, fvs))) + for callback in (self.callbacks + [subscriber.getDbConnector().getDbName()] + [subscriber.getTableName()]): + callback(key, op, dict(fvs)) + + + +def set_node_labels(server): + labels = {} + + version_info = (device_info.get_sonic_version_info() if not UNIT_TESTING + else { "build_version": "20201230.111"}) + dev_data = server.get_db_entry(CONFIG_DB_NAME, 'DEVICE_METADATA', + 'localhost') + dep_type = dev_data['type'] if 'type' in dev_data else "unknown" + + labels["sonic_version"] = version_info['build_version'] + labels["hwsku"] = device_info.get_hwsku() if not UNIT_TESTING else "mock" + labels["deployment_type"] = dep_type + server.mod_db_entry(STATE_DB_NAME, + KUBE_LABEL_TABLE, KUBE_LABEL_SET_KEY, labels) + + +def _update_entry(ct, upd): + # Helper function, to update with case lowered. + ret = dict(ct) + for (k, v) in upd.items(): + ret[k.lower()] = v.lower() + + return ret + + +# +# SERVER-config changes: +# Act if IP or disable changes. +# If disabled or IP removed: +# reset connection +# else: +# join +# Update state-DB appropriately +# +class RemoteServerHandler: + def __init__(self, server): + """ Register self for updates """ + self.server = server + + server.register_handler( + CONFIG_DB_NAME, SERVER_TABLE, self.on_config_update) + self.cfg_server = _update_entry(dflt_cfg_ser, server.get_db_entry( + CONFIG_DB_NAME, SERVER_TABLE, SERVER_KEY)) + + log_debug("startup config: {}".format(str(self.cfg_server))) + + server.register_db(STATE_DB_NAME) + self.st_server = _update_entry(dflt_st_ser, server.get_db_entry( + STATE_DB_NAME, SERVER_TABLE, SERVER_KEY)) + + self.start_time = datetime.datetime.now() + + if not self.st_server[ST_FEAT_UPDATE_TS]: + # This is upon system start. Sleep 10m before join + self.start_time += datetime.timedelta( + seconds=remote_ctr_config[JOIN_LATENCY]) + server.register_timer(self.start_time, self.handle_update) + self.pending = True + log_debug("Pause to join {} seconds @ {}".format( + remote_ctr_config[JOIN_LATENCY], self.start_time)) + else: + self.pending = False + self.handle_update() + + + + def on_config_update(self, key, op, data): + """ On config update """ + if key != SERVER_KEY: + return + + cfg_data = _update_entry(dflt_cfg_ser, data) + if self.cfg_server == cfg_data: + log_debug("No change in server config") + return + + log_debug("Received config update: {}".format(str(data))) + self.cfg_server = cfg_data + + if self.pending: + tnow = datetime.datetime.now() + if tnow < self.start_time: + # Pausing for initial latency since reboot or last retry + due_secs = (self.start_time - tnow).seconds + else: + due_secs = 0 + log_debug("Pending to start in {} seconds at {}".format( + due_secs, self.start_time)) + return + + self.handle_update() + + + def handle_update(self): + # Called upon start and upon any config-update only. + # Called by timer / main thread. + # Main thread calls only if timer is not running. + # + self.pending = False + + ip = self.cfg_server[CFG_SER_IP] + disable = self.cfg_server[CFG_SER_DISABLE] != "false" + + pre_state = dict(self.st_server) + log_debug("server: handle_update: disable={} ip={}".format(disable, ip)) + if disable or not ip: + self.do_reset() + else: + self.do_join(ip, self.cfg_server[ST_SER_PORT], + self.cfg_server[CFG_SER_INSECURE]) + + if pre_state != self.st_server: + self.st_server[ST_SER_UPDATE_TS] = ts_now() + self.server.set_db_entry(STATE_DB_NAME, + SERVER_TABLE, SERVER_KEY, self.st_server) + log_debug("Update server state={}".format(str(self.st_server))) + + + def do_reset(self): + global remote_connected + + kube_commands.kube_reset_master(True) + + self.st_server[ST_SER_CONNECTED] = "false" + log_debug("kube_reset_master called") + + remote_connected = False + + + def do_join(self, ip, port, insecure): + global remote_connected + (ret, out, err) = kube_commands.kube_join_master( + ip, port, insecure) + + if ret == 0: + self.st_server[ST_SER_CONNECTED] = "true" + self.st_server[ST_SER_IP] = ip + self.st_server[ST_SER_PORT] = port + remote_connected = True + + set_node_labels(self.server) + log_debug("kube_join_master succeeded") + else: + # Ensure state reflects the current status + self.st_server[ST_SER_CONNECTED] = "false" + remote_connected = False + + # Join failed. Retry after an interval + self.start_time = datetime.datetime.now() + self.start_time += datetime.timedelta( + seconds=remote_ctr_config[JOIN_RETRY]) + self.server.register_timer(self.start_time, self.handle_update) + self.pending = True + + log_debug("kube_join_master failed retry after {} seconds @{}". + format(remote_ctr_config[JOIN_RETRY], self.start_time)) + + +# +# Feature changes +# +# Handle Set_owner change: +# restart service and/or label add/drop +# +# Handle remote_state change: +# When pending, trigger restart +# +class FeatureTransitionHandler: + def __init__(self, server): + self.server = server + self.cfg_data = defaultdict(lambda: defaultdict(str)) + self.st_data = defaultdict(lambda: defaultdict(str)) + server.register_handler( + CONFIG_DB_NAME, FEATURE_TABLE, self.on_config_update) + server.register_handler( + STATE_DB_NAME, FEATURE_TABLE, self.on_state_update) + return + + + def handle_update(self, feat, set_owner, ct_owner, remote_state): + # Called upon startup once for every feature in config & state DBs. + # There after only called upon changes in either that requires action + # + if not is_systemd_active(feat): + # Nothing todo, if system state is down + return + + label_add = set_owner == "kube" + service_restart = False + + if set_owner == "local": + if ct_owner != "local": + service_restart = True + else: + if (ct_owner != "none") and (remote_state == "pending"): + service_restart = True + + log_debug( + "feat={} set={} ct={} state={} restart={} label_add={}".format( + feat, set_owner, ct_owner, remote_state, service_restart, + label_add)) + # read labels and add/drop if different + self.server.mod_db_entry(STATE_DB_NAME, KUBE_LABEL_TABLE, KUBE_LABEL_SET_KEY, + { "{}_enabled".format(feat): ("true" if label_add else "false") }) + + + # service_restart + if service_restart: + restart_systemd_service(self.server, feat, set_owner) + + + + def on_config_update(self, key, op, data): + # Hint/Note: + # If the key don't pre-exist: + # This attempt to read will create the key for given + # field with empty string as value and return empty string + + init = key not in self.cfg_data + old_set_owner = self.cfg_data[key][CFG_FEAT_OWNER] + + self.cfg_data[key] = _update_entry(dflt_cfg_feat, data) + set_owner = self.cfg_data[key][CFG_FEAT_OWNER] + + if (not init) and (old_set_owner == set_owner): + # No change, bail out + log_debug("No change in feat={} set_owner={}. Bail out.".format( + key, set_owner)) + return + + if key in self.st_data: + log_debug("{} init={} old_set_owner={} owner={}".format(key, init, old_set_owner, set_owner)) + self.handle_update(key, set_owner, + self.st_data[key][ST_FEAT_OWNER], + self.st_data[key][ST_FEAT_REMOTE_STATE]) + + else: + # Handle it upon data from state-db, which must come + # if systemd is up + log_debug("Yet to get state info key={}. Bail out.".format(key)) + return + + + def on_state_update(self, key, op, data): + # Hint/Note: + # If the key don't pre-exist: + # This attempt to read will create the key for given + # field with empty string as value and return empty string + + init = key not in self.st_data + old_remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE] + + self.st_data[key] = _update_entry(dflt_st_feat, data) + remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE] + + if (not init) and ( + (old_remote_state == remote_state) or (remote_state != "pending")): + # no change or nothing to do. + return + + if key in self.cfg_data: + log_debug("{} init={} old_remote_state={} remote_state={}".format(key, init, old_remote_state, remote_state)) + self.handle_update(key, self.cfg_data[key][CFG_FEAT_OWNER], + self.st_data[key][ST_FEAT_OWNER], + remote_state) + return + + +# +# Label re-sync +# When labels applied at remote are pending, start monitoring +# API server reachability and trigger the label handler to +# apply the labels +# +# Non-empty KUBE_LABELS|PENDING, monitor API Server reachability +# +class LabelsPendingHandler: + def __init__(self, server): + server.register_handler(STATE_DB_NAME, KUBE_LABEL_TABLE, self.on_update) + self.server = server + self.pending = False + self.set_labels = {} + return + + + def on_update(self, key, op, data): + # For any update sync with kube API server. + # Don't optimize, as API server could differ with DB's contents + # in many ways. + # + if (key == KUBE_LABEL_SET_KEY): + self.set_labels = dict(data) + else: + return + + if remote_connected and not self.pending: + self.update_node_labels() + else: + log_debug("Skip label update: connected:{} pending:{}". + format(remote_connected, self.pending)) + + + + def update_node_labels(self): + # Called on config update by main thread upon init or config-change or + # it was not connected during last config update + # NOTE: remote-server-handler forces a config update notification upon + # join. + # Or it could be called by timer thread, if last upate to API server + # failed. + # + self.pending = False + ret = kube_commands.kube_write_labels(self.set_labels) + if (ret != 0): + self.pending = True + pause = remote_ctr_config[LABEL_RETRY] + ts = (datetime.datetime.now() + datetime.timedelta(seconds=pause)) + self.server.register_timer(ts, self.update_node_labels) + + log_debug("ret={} set={} pending={}".format(ret, + str(self.set_labels), self.pending)) + return + + +def main(): + init() + server = MainServer() + RemoteServerHandler(server) + FeatureTransitionHandler(server) + LabelsPendingHandler(server) + server.run() + print("ctrmgrd.py main called") + return 0 + + +if __name__ == '__main__': + if os.geteuid() != 0: + exit("Please run as root. Exiting ...") + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service new file mode 100644 index 0000000000..6052c10785 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service @@ -0,0 +1,14 @@ +[Unit] +Description=Container Manager watcher daemon +Requires=updategraph.service +After=updategraph.service + + +[Service] +Type=simple +ExecStart=/usr/local/bin/ctrmgrd.py +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py new file mode 100755 index 0000000000..1ebfa606f0 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import fcntl +import inspect +import json +import os +import shutil +import ssl +import subprocess +import sys +import syslog +import tempfile +import urllib.request +from urllib.parse import urlparse + +import yaml +from sonic_py_common import device_info + +KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf" +KUBELET_YAML = "/var/lib/kubelet/config.yaml" +SERVER_ADMIN_URL = "https://{}/admin.conf" +LOCK_FILE = "/var/lock/kube_join.lock" + +# kubectl --kubeconfig label nodes +#